{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import pickle\n",
    "import time\n",
    "#from tensorflow.python.layers import core as layers_core\n",
    "#from tensorflow.contrib.seq2seq import ScheduledEmbeddingTrainingHelper\n",
    "#from tensorflow.contrib.seq2seq import AttentionWrapper, AttentionWrapperState\n",
    "#from tensorflow.python.ops import random_ops\n",
    "#from tensorflow.python.ops import array_ops\n",
    "#from tensorflow.python.ops import math_ops\n",
    "#from tensorflow.python.ops import control_flow_ops\n",
    "#from tensorflow.python.ops import check_ops\n",
    "#from tensorflow.python.framework import ops\n",
    "#from tensorflow.python.framework import dtypes\n",
    "from functools import partial\n",
    "from preprocess_library import SentenceBatchGenerator, Word2Numb, bin_batch_create\n",
    "import bisect\n",
    "from tqdm import tqdm\n",
    "import argparse\n",
    "from threading import Thread\n",
    "import math\n",
    "import matplotlib.pyplot as plt\n",
    "#import winsound"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Sets which GPU to use\n",
    "gpu_num = 1\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=str(gpu_num)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Config(object):\n",
    "    \"\"\"The model configuration\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self,\n",
    "                 chan_enc_layers, #Could be a scalar(num of hidden layers) or a list(dims of hidden layers)\n",
    "                 chan_dec_layers, #Could be a scalar(num of hidden layers) or a list(dims of hidden layers)\n",
    "                 \n",
    "                 numb_tx_bits, # Could be a list for variable length\n",
    "                 chan_dec_out_len, #==chan_enc_in_len\n",
    "                 \n",
    "                 channel={'type':'erasure','chan_param':0.9},\n",
    "                 num_epochs=100,\n",
    "                 learn_rate=0.001,\n",
    "                \n",
    "                 enc_hidden_act = tf.nn.relu,\n",
    "                 dec_hidden_act = tf.nn.relu,\n",
    "                 iter_save_rate = 10000,\n",
    "                 model_save_path = \"checkpoints/\",\n",
    "                 save_max = 5,\n",
    "                 \n",
    "                 batch_size = 1,\n",
    "                 batch_size_test = 128,\n",
    "                 length_from=4,\n",
    "                 length_to=30,\n",
    "                 bin_len = 4,\n",
    "                 bits_per_bin = [200,250,300,350,400,450,500],\n",
    "                 \n",
    "                 dataset = 'euro',\n",
    "                 w2n_path=\"../data/w2n_n2w_TopEuro.pickle\",\n",
    "#                 traindata_path=\"../../data/training_euro_wordlist20.pickle\",\n",
    "                 traindata_path=\"../data/corpora/europarl-v7.en/europarl-v7.en\",\n",
    "#                 testdata_path=\"../../data/testing_euro_wordlist20.pickle\",\n",
    "                 testdata_path=\"../data/corpora/europarl-v7.en/europarl-v7.en\",\n",
    "                 embed_path=\"../data/200_embed_large_TopEuro.pickle\",\n",
    "                 max_test_counter = int(1e6),\n",
    "                 max_validate_counter = 10000,\n",
    "                 max_batch_in_epoch = 1000,\n",
    "                 iter_print_rate = None):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            model_save_path - path where the model is saved\n",
    "            channel - dict with keys type [erasure,awgn] and chan_param\n",
    "            bits_per_bin - ensure this is even and of the same size as the number of batches \n",
    "        \"\"\"\n",
    "        \n",
    "        if isinstance(chan_enc_layers, list):\n",
    "            self.enc_layers_dims = chan_enc_layers\n",
    "            self.num_enc_layers = len(chan_enc_layers)\n",
    "        else:\n",
    "            self.num_enc_layers = chan_enc_layers\n",
    "            self.enc_layers_dims = None\n",
    "\n",
    "        if isinstance(chan_dec_layers, list):\n",
    "            self.dec_layers_dims = chan_dec_layers\n",
    "            self.num_dec_layers = len(chan_dec_layers)\n",
    "        else:\n",
    "            self.num_dec_layers = chan_dec_layers\n",
    "            self.dec_layers_dims = None\n",
    "        \n",
    "        self.enc_hidden_act = enc_hidden_act\n",
    "        self.dec_hidden_act = dec_hidden_act\n",
    "        self.epochs = num_epochs\n",
    "        self.learn_rate = learn_rate\n",
    "        self.chan_dec_out_len = chan_dec_out_len\n",
    "        self.chan_enc_in_len = chan_dec_out_len\n",
    "        \n",
    "        #batch properties\n",
    "        self.batch_size = batch_size\n",
    "        \n",
    "        self.channel = channel\n",
    "        self.numb_tx_bits = numb_tx_bits\n",
    "        self.iter_save_rate = iter_save_rate\n",
    "        self.model_save_path = model_save_path\n",
    "        \n",
    "        self.traindata_path = traindata_path\n",
    "        self.testdata_path = testdata_path\n",
    "        self.max_test_counter = max_test_counter\n",
    "        self.max_batch_in_epoch = max_batch_in_epoch\n",
    "        self.iter_print_rate = max_batch_in_epoch if iter_print_rate == None else iter_print_rate\n",
    "        self.save_max = save_max"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ChannelEncoder(object):\n",
    "    \"\"\"\n",
    "    Channel encoder object. \n",
    "    Takes in a bit vector of compressed states from a stacked BLSTM. \n",
    "    Outputs a bit vector of the specified length to be passed through the channel, with redundancy added through an\n",
    "    MLP network\n",
    "    \"\"\"\n",
    "    def __init__(self, chan_enc_input, isTrain, config):\n",
    "        \n",
    "        self.config = config\n",
    "        self.input = chan_enc_input\n",
    "        self.input_len = config.chan_enc_in_len\n",
    "        self.batch_size = config.batch_size\n",
    "        self.isTrain = isTrain\n",
    "        \n",
    "        self.reduced_states = self.build_enc_net()\n",
    "        print(\"reduced states:\", self.reduced_states)\n",
    "        self.output = self.binarize(self.reduced_states)\n",
    "        \n",
    "        print(\"Output from channel encoder: \", self.output)\n",
    "    \n",
    "    def get_net_dims(self, n_in, n_out):\n",
    "        \n",
    "        if (self.config.enc_layers_dims != None):\n",
    "            return self.config.enc_layers_dims\n",
    "        \n",
    "        num_hidden_layers = self.config.num_enc_layers #num_hidden >= 2\n",
    "        hidden_dims = []\n",
    "        \n",
    "        \"\"\"\n",
    "        Uses ideas from D.Stathakis' paper to determine the number of neurons for first 2 hidden layers\n",
    "        \"\"\"\n",
    "        n_h1 = int(math.sqrt((n_out + 2) * n_in) + 2 * math.sqrt(n_in / (n_out + 2)))\n",
    "        n_h2 = int(n_out * math.sqrt(n_in / (n_out + 2)))\n",
    "\n",
    "        hidden_dims.append(n_h1)\n",
    "        if (self.config.num_enc_layers > 1):\n",
    "            hidden_dims.append(n_h2)\n",
    "        \n",
    "        #For remaining layers\n",
    "        \n",
    "        for _ in range(num_hidden_layers - 2):\n",
    "            \n",
    "            hidden_dims.append(int(hidden_dims[-1] * 0.8))\n",
    "            \n",
    "        \n",
    "        return hidden_dims\n",
    "    \n",
    "    def build_enc_net(self):\n",
    "        \"\"\"\n",
    "        Builds the channel encoder\n",
    "        \"\"\"\n",
    "        print(self.input_len, self.config.numb_tx_bits)\n",
    "        hidden_dims = self.get_net_dims(self.input_len, self.config.numb_tx_bits) #length >= 1\n",
    "   \n",
    "        print(\"Hidden encoder dimensions: \", hidden_dims)\n",
    "\n",
    "        hidden_layers = [] #list of tensors for storing activations from each hidden layer\n",
    "        \n",
    "        print(\"Input dims:\", self.input)\n",
    "\n",
    "        print(\"Test\", self.input.dtype.base_dtype)\n",
    "    \n",
    "        hidden_act1 = tf.layers.dense(self.input, hidden_dims[0], \n",
    "                                      activation = self.config.enc_hidden_act,\n",
    "                                      kernel_initializer = tf.contrib.layers.xavier_initializer(),\n",
    "                                      name = \"chan_enc_h1\") \n",
    "        \n",
    "        print(\"hidden1\", hidden_act1)\n",
    "        hidden_layers.append(hidden_act1)\n",
    "        \n",
    "        for i in range(1, len(hidden_dims)):\n",
    "        \n",
    "#             hidden_act = tf.contrib.layers.fully_connected(hidden_layers[-1], \n",
    "#                                                            hidden_dims[i], \n",
    "#                                                            activation_fn = self.config.enc_hidden_act) \n",
    "            hidden_act = tf.layers.dense(hidden_layers[-1], hidden_dims[i], activation = self.config.enc_hidden_act,\n",
    "                                         kernel_initializer = tf.contrib.layers.xavier_initializer(),\n",
    "                                         name = \"chan_enc_h\" + str(i + 1)) \n",
    "            hidden_layers.append(hidden_act)\n",
    "        \n",
    "        if (len(hidden_dims) > 1):\n",
    "            #Skip connection between h1 and output feed\n",
    "            output_feed = tf.concat([hidden_act1, hidden_layers[-1]], axis = 1)\n",
    "            print(\"Encoder test:\", output_feed)\n",
    "        else:\n",
    "            output_feed = hidden_layers[-1]\n",
    "            \n",
    "        b_norm_output_feed = tf.contrib.layers.batch_norm(output_feed)\n",
    "        \n",
    "        out_states = tf.layers.dense(b_norm_output_feed, self.config.numb_tx_bits, activation = tf.tanh,\n",
    "                                     kernel_initializer = tf.contrib.layers.xavier_initializer(),\n",
    "                                     name = \"chan_enc_out\") \n",
    "        return out_states\n",
    "        \n",
    "    def training_binarizer(self, input_layer):\n",
    "        \"\"\"Binarizer function used at training\n",
    "        \"\"\"\n",
    "        prob = tf.truediv(tf.add(1.0, input_layer), 2.0)\n",
    "        bernoulli = tf.contrib.distributions.Bernoulli(probs=prob, dtype=tf.float32)\n",
    "        return 2 * bernoulli.sample() - 1\n",
    "\n",
    "    def test_binarizer(self, input_layer):\n",
    "        \"\"\"Binarizer function used during testing\n",
    "        \"\"\"\n",
    "        ones = tf.ones_like(input_layer,dtype=tf.float32)\n",
    "        neg_ones = tf.scalar_mul(-1.0, ones)\n",
    "        return tf.where(tf.less(input_layer,0.0), neg_ones, ones)\n",
    "    \n",
    "    def binarize(self, reduced_states):\n",
    "        binarized = tf.cond(self.isTrain,\n",
    "                            partial(self.training_binarizer, reduced_states),\n",
    "                            partial(self.test_binarizer, reduced_states))\n",
    "        \n",
    "        pass_through = tf.identity(reduced_states) # this is used for pass through gradient back prop\n",
    "        return pass_through + tf.stop_gradient(binarized - pass_through)\n",
    "    \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Channel(object):\n",
    "    \"\"\"\n",
    "    Takes in embedding vectors converted to bits, and models the impact of a channel on them\n",
    "    \"\"\"\n",
    "    def __init__(self, channel_in, config):\n",
    "        self.config = config\n",
    "        self.chan_type = config.channel[\"type\"] #dict of items type and chan_param\n",
    "        self.chan_param = config.channel[\"chan_param\"]\n",
    "        self.input = channel_in\n",
    "        self.output = self.build_channel()\n",
    "        print(\"Channel Output: \", self.output)\n",
    "        \n",
    "    def gaussian_noise_layer(self, input_layer, std, name=None):\n",
    "        noise = tf.random_normal(shape=tf.shape(input_layer), mean=0.0,\n",
    "                                 stddev=std, dtype=tf.float32, name=name)\n",
    "        return input_layer + noise\n",
    "\n",
    "    def test_binarizer(self, input_layer):\n",
    "        \"\"\"Binarizer function used during testing\n",
    "        \"\"\"\n",
    "        ones = tf.ones_like(input_layer,dtype=tf.float32)\n",
    "        neg_ones = tf.scalar_mul(-1.0, ones)\n",
    "        return tf.where(tf.less(input_layer,0.0), neg_ones, ones)\n",
    "    \n",
    "    def build_channel(self):\n",
    "        \"\"\"Build the final binarization layer and the channel\n",
    "        \"\"\"\n",
    "        # if no channel, just output the encoder states\n",
    "        if self.chan_type == \"none\":\n",
    "            chan_output = self.input\n",
    "            \n",
    "        elif self.chan_type == \"erasure\":\n",
    "            chan_output = tf.nn.dropout(self.input,\n",
    "                                         keep_prob=self.chan_param,\n",
    "                                         name=\"erasure_chan_dropout_ch\")\n",
    "                            \n",
    "        elif self.chan_type == \"awgn\":\n",
    "            chan_output = self.gaussian_noise_layer(self.channel_in,\n",
    "                                             std=self.chan_param,\n",
    "                                             name=\"awgn_chan_noise\")\n",
    "            \n",
    "        elif self.type == 'bsc':\n",
    "            \n",
    "            chan_output = tf.where(tf.greater(tf.random_uniform(shape=tf.shape(self.channel_in)),\n",
    "                                              self.chan_param),\n",
    "                                   self.channel_in,\n",
    "                                   -self.channel_in)\n",
    "            \n",
    "        else:\n",
    "            raise NameError('Channel type is not known.')\n",
    "\n",
    "        return chan_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ChannelDecoder(object):\n",
    "    \"\"\"\n",
    "    A decoder that uses a fully connected net to convert channel output to original channel encoding\n",
    "    Takes in a bit vector from the channel\n",
    "    Outputs a vector of same length as input to chann encoder on the other side of channel, with (ideally) identical elements\n",
    "    \n",
    "    \"\"\"\n",
    "    def __init__(self, chan_out, isTrain, config):\n",
    "        \n",
    "        self.config = config\n",
    "        self.dec_input = chan_out\n",
    "        self.dec_inp_len = config.numb_tx_bits\n",
    "        self.output_len = config.chan_dec_out_len\n",
    "        self.batch_size = config.batch_size\n",
    "        self.num_hidden_layers = config.num_dec_layers\n",
    "        self.isTrain = isTrain\n",
    "        self.dec_network_out = self.build_dec_net()\n",
    "      \n",
    "        self.bin_pred = tf.where(tf.less(self.dec_network_out,0.0), \n",
    "                                 -1 * tf.ones_like(self.dec_network_out,dtype=tf.float32), \n",
    "                                 tf.ones_like(self.dec_network_out,dtype=tf.float32))\n",
    "    \n",
    "    def get_net_dims(self, n_in, n_out):\n",
    "        \n",
    "        layer_drop_const = 0.95\n",
    "        \n",
    "        if (self.config.dec_layers_dims != None):\n",
    "            return self.config.dec_layers_dims\n",
    "        \n",
    "        num_hidden_layers = self.config.num_dec_layers #num_hidden >= 1\n",
    "        hidden_dims = []\n",
    "        \n",
    "        \"\"\"\n",
    "        Uses ideas from D.Stathakis' paper to determine the number of neurons for first 2 hidden layers\n",
    "        \"\"\"\n",
    "        n_h1 = int(math.sqrt((n_out + 2) * n_in) + 2 * math.sqrt(n_in / (n_out + 2)))\n",
    "        n_h2 = int(n_out * math.sqrt(n_in / (n_out + 2)))\n",
    "\n",
    "        hidden_dims.append(n_h1)\n",
    "        if (self.config.num_dec_layers > 1):\n",
    "            hidden_dims.append(n_h2)\n",
    "        \n",
    "        #For remaining layers\n",
    "        \n",
    "        for _ in range(num_hidden_layers - 2):\n",
    "            \n",
    "            hidden_dims.append(int(hidden_dims[-1] * layer_drop_const))\n",
    "            \n",
    "        \n",
    "        return hidden_dims\n",
    "    \n",
    "    \n",
    "    def build_dec_net(self):\n",
    "        \"\"\"\n",
    "        Builds the channel encoder\n",
    "        \"\"\"\n",
    " \n",
    "        hidden_dims = self.get_net_dims(self.dec_inp_len, self.output_len) \n",
    "        #hidden_dims = [2048, 1024, 512, 256, 128]\n",
    "        print(\"Hidden decoder dimensions: \", hidden_dims)\n",
    "        hidden_layers = [] #list of tensors for storing activations from each hidden layer\n",
    "        \n",
    "        hidden_act1 = tf.layers.dense(self.dec_input, hidden_dims[0], \n",
    "                                      activation = self.config.dec_hidden_act, \n",
    "                                      kernel_initializer = tf.contrib.layers.xavier_initializer(), \n",
    "                                      name = \"chan_dec_h1\") \n",
    "\n",
    "        hidden_layers.append(hidden_act1)\n",
    "        \n",
    "        for i in range(1, len(hidden_dims)):\n",
    "        \n",
    "            hidden_act = tf.layers.dense(hidden_layers[-1],\n",
    "                                         hidden_dims[i], \n",
    "                                         activation = self.config.dec_hidden_act,\n",
    "                                         kernel_initializer = tf.contrib.layers.xavier_initializer(),\n",
    "                                         name = \"chan_dec_h\" + str(i + 1)) \n",
    "\n",
    "            hidden_layers.append(hidden_act)\n",
    "        \n",
    "        if (len(hidden_dims) > 1):\n",
    "            #Skip connection between h1 and output feed\n",
    "            output_feed = tf.concat([hidden_act1, hidden_layers[-1]], axis = 1)\n",
    "            print(\"Decoder test:\", output_feed)\n",
    "        else:\n",
    "            output_feed = hidden_layers[-1]    \n",
    "\n",
    "        out_states = tf.layers.dense(output_feed, \n",
    "                                     self.output_len, \n",
    "                                     activation = None, \n",
    "                                     kernel_initializer = tf.contrib.layers.xavier_initializer(),\n",
    "                                     name = \"chan_dec_out\")\n",
    "    \n",
    "        print(\"Dec out\", out_states)                                               \n",
    "        return out_states\n",
    "        \n",
    "\n",
    "    def training_binarizer(self, input_layer):\n",
    "        \"\"\"Binarizer function used at training\n",
    "        \"\"\"\n",
    "        prob = tf.truediv(tf.add(1.0, input_layer), 2.0)\n",
    "        bernoulli = tf.contrib.distributions.Bernoulli(probs=prob, dtype=tf.float32)\n",
    "        return 2 * bernoulli.sample() - 1\n",
    "\n",
    "    def test_binarizer(self, input_layer):\n",
    "        \"\"\"Binarizer function used during testing\n",
    "        \"\"\"\n",
    "        ones = tf.ones_like(input_layer,dtype=tf.float32)\n",
    "        neg_ones = tf.scalar_mul(-1.0, ones)\n",
    "        return tf.where(tf.less(input_layer,0.0), neg_ones, ones)\n",
    "\n",
    "    def binarize(self,input_layer):\n",
    "        \"\"\"This part of the code binarizes the reduced states. The last line ensure the\n",
    "        backpropagation gradients pass through the binarizer unchanged\n",
    "        \"\"\"\n",
    "        binarized = tf.cond(self.isTrain,\n",
    "                            partial(self.training_binarizer, input_layer),\n",
    "                            partial(self.test_binarizer, input_layer))\n",
    "\n",
    "        pass_through = tf.identity(input_layer) # this is used for pass through gradient back prop\n",
    "        return pass_through + tf.stop_gradient(binarized - pass_through )\n",
    "        \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Previous learn_rate policy\n",
    "class ChannelSystem(object):\n",
    "    \"\"\"\n",
    "    Generates a channel coding model with encoder + channel + decoder.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, config, train_size, test_size):\n",
    "        self.config = config\n",
    "        self.training_counter = 0\n",
    "        self.test_counter = 0\n",
    "        #self.train_data = train_data\n",
    "        #self.test_data = test_data\n",
    "        self.train_size = train_size\n",
    "        self.test_size = test_size \n",
    "        # ==== reset graph ====\n",
    "        tf.reset_default_graph()\n",
    "          \n",
    "        self.chan_enc_in_len = config.chan_enc_in_len\n",
    "        self.chan_dec_out_len = config.chan_dec_out_len\n",
    "        \n",
    "        #===Legacy placeholders====\n",
    "        #self.chan_enc_inputs = tf.placeholder_with_default(queue_vars['enc_inputs'],shape=(config.batch_size, None), name='encoder_inputs')\n",
    "        #self.chan_enc_inputs_len = tf.placeholder_with_default(queue_vars['enc_inputs_len'],shape=(None,), name='encoder_inputs_length')\n",
    "        #self.dec_inputs = tf.placeholder_with_default(queue_vars['dec_inputs'],shape=(config.batch_size, None), name='decoder_targets')\n",
    "        #self.dec_targets = tf.placeholder_with_default(queue_vars['dec_targets'],shape=(config.batch_size, None), name='decoder_targets')\n",
    " \n",
    "        \n",
    "        # ======= Placeholders =======        \n",
    "        self.chan_enc_inputs = tf.placeholder(tf.float32, shape = [None, self.chan_enc_in_len], name = 'enc_inputs')\n",
    "        self.dec_targets = tf.placeholder(tf.float32, shape = [None, self.chan_dec_out_len], name = 'dec_targets')\n",
    "        self.isTrain = tf.placeholder(tf.bool,shape=(), name='isTrain')\n",
    "\n",
    "        self.learn_rate = tf.placeholder(tf.float32,shape=(), name='learn_rate')\n",
    "        \n",
    "        self.X = tf.placeholder(tf.float32, shape = [None, self.chan_enc_in_len], name = 'dataset_iter_X')\n",
    "        self.Y = tf.placeholder(tf.float32, shape = [None, self.chan_dec_out_len], name = 'dataset_iter_Y')\n",
    "        print(\"X:\", self.X)  \n",
    "        self.dataset = tf.data.Dataset.from_tensor_slices((self.X, self.Y)).shuffle(buffer_size=100).batch(config.batch_size).repeat()\n",
    "        \n",
    "        self.iterator = self.dataset.make_initializable_iterator()\n",
    "        \n",
    "        self.chan_enc_inputs, self.dec_targets = self.iterator.get_next()\n",
    "        # ==== Building neural network graph ====\n",
    "        \n",
    "        \n",
    "        self.chan_encoder = ChannelEncoder(self.chan_enc_inputs, \n",
    "                                       self.isTrain,\n",
    "                                       self.config)\n",
    "\n",
    "        self.channel = Channel(self.chan_encoder.output,\n",
    "                               self.config)\n",
    "        \n",
    "        self.decoder = ChannelDecoder(self.channel.output,\n",
    "                                     self.isTrain,\n",
    "                                     self.config)\n",
    "\n",
    "        \n",
    "        # ==== define loss and training op and accuracy ====\n",
    "        \n",
    "        self.accuracy = self.define_accuracy()\n",
    "        self.loss, self.train_op = self.define_loss()\n",
    "\n",
    "        # ==== set up training/updating procedure ====\n",
    "        self.saver = tf.train.Saver(max_to_keep = self.config.save_max)\n",
    "\n",
    "\n",
    "#     def load_trained_model(self, sess, model_save_path):\n",
    "#         \"\"\"\n",
    "#         Loads a trained model from what was saved. Insert the trained model path\n",
    "#         \"\"\"\n",
    "#         meta_file_path = model_save_path + \".meta\"\n",
    "        \n",
    "#         new_saver = tf.train.import_meta_graph(meta_file_path) #Loads graph\n",
    "#         new_saver.restore(sess, tf.train.latest_checkpoint('checkpoints/'))\n",
    "        \n",
    "#         #Test\n",
    "#         variables_names = [v.name for v in tf.trainable_variables()]\n",
    "#         values = sess.run(variables_names)\n",
    "#         print(\"Printing restored variables:\")\n",
    "#         for k, v in zip(variables_names, values):\n",
    "#             print (\"Variable: \", k)\n",
    "#             print (\"Shape: \", v.shape)\n",
    "#             print (v)\n",
    "\n",
    "    def define_accuracy(self):\n",
    "        eq_indicator = tf.cast(tf.equal(self.decoder.bin_pred, self.dec_targets), dtype=tf.float32)\n",
    "        return tf.reduce_mean(eq_indicator)\n",
    "\n",
    "    def define_loss(self):\n",
    "        \n",
    "        adapted_targets = tf.cast(self.dec_targets > 0, tf.float32)\n",
    "        loss = tf.losses.hinge_loss(adapted_targets, self.decoder.dec_network_out)\n",
    "\n",
    "        # train it\n",
    "        train_op = tf.train.AdamOptimizer(learning_rate=self.learn_rate).minimize(loss)\n",
    "        return loss, train_op\n",
    "    \n",
    "    \"\"\"\n",
    "    Script to generate train and test data\n",
    "    \"\"\"\n",
    "    def gen_bin_data(self, num_samples, sample_len, seed_num = None):\n",
    "        \n",
    "        if (seed_num != None):\n",
    "            np.random.seed(seed_num)\n",
    "        randMat = np.random.rand(num_samples, sample_len)\n",
    "        binMat = np.where(randMat > 0.5, 1.0, -1.0)\n",
    "        return binMat\n",
    "\n",
    "          \n",
    "    def train_and_test(self, sess, grammar=None):\n",
    "        \"\"\"\n",
    "        Trains the model\n",
    "        \"\"\"\n",
    "        try:\n",
    "            n_epochs = self.config.epochs\n",
    "            batch_sz = self.config.batch_size\n",
    "            max_batch = self.config.max_batch_in_epoch\n",
    "\n",
    "            sess.run(tf.global_variables_initializer())\n",
    "\n",
    "            #Train on train data\n",
    "            print(\"Training...\")\n",
    "            tic = time.time()\n",
    "\n",
    "            accuracies = []\n",
    "            learn_rates = []\n",
    "        \n",
    "            #train_data_gen = self.gen_bin_data(self.train_size, self.chan_enc_in_len)\n",
    "\n",
    "            curr_lr = self.config.learn_rate\n",
    "            \n",
    "            for epoch in range(n_epochs):\n",
    "                train_data = self.gen_bin_data(self.train_size, self.chan_enc_in_len)\n",
    "                sess.run(self.iterator.initializer, feed_dict = {self.X: train_data, self.Y: train_data})\n",
    "                len_acc = len(accuracies)\n",
    "                if (len_acc > 2 and epoch % 2 == 0):\n",
    "                    if ((len_acc <= 10 and np.mean(accuracies[-3:-1]) > accuracies[-1]) or \n",
    "                        (len_acc in range(11, 51) and np.mean(accuracies[-6:-2]) > np.mean(accuracies[-2:])) or \n",
    "                        (len_acc in range(51, 101) and np.mean(accuracies[-10:-4]) > np.mean(accuracies[-4:])) or \n",
    "                        (len_acc in range(101, 201) and np.mean(accuracies[-30:-10]) > np.mean(accuracies[-10:])) or\n",
    "                        (len_acc > 200 and np.mean(accuracies[-50:-10]) > np.mean(accuracies[-10:]))):\n",
    "                            \n",
    "                            if (curr_lr > 1e-12):\n",
    "                                curr_lr = curr_lr / np.sqrt(2)\n",
    "                learn_rates.append(curr_lr)\n",
    "                for i in range(max_batch):               \n",
    "\n",
    "                    train_fd = {self.isTrain: True, self.learn_rate: curr_lr}       \n",
    "                    _, loss = sess.run([self.train_op, self.loss], feed_dict = train_fd)\n",
    "\n",
    "                    self.training_counter += 1\n",
    "\n",
    "                    if (self.training_counter % self.config.iter_print_rate == 0):\n",
    "\n",
    "                        toc = time.time()\n",
    "\n",
    "                        acc = sess.run(self.accuracy, train_fd)\n",
    "\n",
    "                        print(\"Epoch: \", epoch + 1, \n",
    "                              \"Accuracy: \", acc,\n",
    "                              \"Training iteration: \", self.training_counter,\n",
    "                              \"Training time: \", int(toc-tic), \"s\",\n",
    "                              \"Training loss: \", loss,\n",
    "                              \"Learning rate\", curr_lr)\n",
    "                        accuracies.append(acc)\n",
    "\n",
    "                    if self.training_counter % self.config.iter_save_rate == 0:\n",
    "                        if (len(accuracies) > 1 and accuracies[-1] > np.amax(accuracies[:-1])):\n",
    "                            self.saver.save(sess, self.config.model_save_path, global_step=self.training_counter)\n",
    "                            print(\"Model saved in file: %s\" % self.config.model_save_path)\n",
    "\n",
    "        except KeyboardInterrupt:\n",
    "            print(\"Training interrupted\")\n",
    "            \n",
    "        #Save model, plot accuracies    \n",
    "        self.saver.save(sess, self.config.model_save_path, global_step=self.training_counter)\n",
    "        print(\"Model saved in file: %s\" % self.config.model_save_path) \n",
    "        \n",
    "        plt.plot(accuracies)\n",
    "        plt.title(\"Accuracies vs epochs\")\n",
    "        plt.xlabel('epochs')\n",
    "        plt.ylabel('Acc')\n",
    "        plt.grid(True)               \n",
    "            \n",
    "        #Test on test data    \n",
    "        test_data = self.gen_bin_data(self.test_size, self.chan_enc_in_len)\n",
    "\n",
    "        sess.run(self.iterator.initializer, feed_dict = {self.X: test_data, self.Y: test_data})\n",
    "        print(\"Testing...\")\n",
    "        \n",
    "        test_fd = {self.isTrain: False, self.learn_rate: self.config.learn_rate} \n",
    "        test_loss = sess.run(self.loss, feed_dict = test_fd)\n",
    "        \n",
    "        test_acc = sess.run(self.accuracy, test_fd)\n",
    "        \n",
    "        print(\"Test loss: \", test_loss, \"\\nTest accuracy: \", test_acc)\n",
    "            \n",
    "        return (accuracies, test_acc, learn_rates)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_model(config, train_size, test_size):\n",
    "   \n",
    "    channel_sys = ChannelSystem(chan_enc_config, train_size,test_size)\n",
    "    with tf.Session() as sess:\n",
    "        train_acc, test_acc, learn_rates = channel_sys.train_and_test(sess)\n",
    "    \n",
    "    return (train_acc, test_acc, learn_rates)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X: Tensor(\"dataset_iter_X:0\", shape=(?, 400), dtype=float32)\n",
      "400 445.0\n",
      "Hidden encoder dimensions:  [4096, 2048, 1024, 512]\n",
      "Input dims: Tensor(\"IteratorGetNext:0\", shape=(?, 400), dtype=float32)\n",
      "Test <dtype: 'float32'>\n",
      "hidden1 Tensor(\"chan_enc_h1/Relu:0\", shape=(?, 4096), dtype=float32)\n",
      "Encoder test: Tensor(\"concat:0\", shape=(?, 4608), dtype=float32)\n",
      "reduced states: Tensor(\"chan_enc_out/Tanh:0\", shape=(?, 445), dtype=float32)\n",
      "Output from channel encoder:  Tensor(\"add:0\", shape=(?, 445), dtype=float32)\n",
      "Channel Output:  Tensor(\"erasure_chan_dropout_ch/mul:0\", shape=(?, 445), dtype=float32)\n",
      "Hidden decoder dimensions:  [4096, 2048, 1600, 1024, 512]\n",
      "Decoder test: Tensor(\"concat_1:0\", shape=(?, 4608), dtype=float32)\n",
      "Dec out Tensor(\"chan_dec_out/BiasAdd:0\", shape=(?, 400), dtype=float32)\n",
      "Training...\n",
      "Epoch:  1 Accuracy:  0.846725 Training iteration:  1000 Training time:  43 s Training loss:  0.37268272 Learning rate 8.003998000000001e-05\n",
      "Epoch:  2 Accuracy:  0.849745 Training iteration:  2000 Training time:  86 s Training loss:  0.36380547 Learning rate 6.005998000000001e-05\n",
      "Model saved in file: checkpoints/test_model\n",
      "Epoch:  3 Accuracy:  0.850595 Training iteration:  3000 Training time:  130 s Training loss:  0.36372906 Learning rate 4.0079979999999994e-05\n",
      "Model saved in file: checkpoints/test_model\n",
      "Epoch:  4 Accuracy:  0.8496675 Training iteration:  4000 Training time:  174 s Training loss:  0.3615212 Learning rate 2.0099980000000013e-05\n",
      "Epoch:  5 Accuracy:  0.85032 Training iteration:  5000 Training time:  217 s Training loss:  0.36179242 Learning rate 1.1998000000000423e-07\n",
      "Epoch:  6 Accuracy:  0.8490475 Training iteration:  6000 Training time:  260 s Training loss:  0.36386898 Learning rate 2.0060019999999996e-05\n",
      "Epoch:  7 Accuracy:  0.848835 Training iteration:  7000 Training time:  304 s Training loss:  0.3670706 Learning rate 4.004002e-05\n",
      "Epoch:  8 Accuracy:  0.846195 Training iteration:  8000 Training time:  347 s Training loss:  0.36927727 Learning rate 6.002002000000001e-05\n",
      "Epoch:  9 Accuracy:  0.8465175 Training iteration:  9000 Training time:  390 s Training loss:  0.37089196 Learning rate 8.000002000000001e-05\n",
      "Epoch:  10 Accuracy:  0.8468525 Training iteration:  10000 Training time:  433 s Training loss:  0.36884323 Learning rate 9.998002e-05\n",
      "Epoch:  11 Accuracy:  0.847925 Training iteration:  11000 Training time:  477 s Training loss:  0.3668596 Learning rate 8.003997999999998e-05\n",
      "Epoch:  12 Accuracy:  0.849175 Training iteration:  12000 Training time:  520 s Training loss:  0.36534464 Learning rate 6.005998000000001e-05\n",
      "Epoch:  13 Accuracy:  0.848635 Training iteration:  13000 Training time:  563 s Training loss:  0.36423495 Learning rate 4.0079979999999994e-05\n",
      "Epoch:  14 Accuracy:  0.8494975 Training iteration:  14000 Training time:  606 s Training loss:  0.36292103 Learning rate 2.0099980000000013e-05\n",
      "Epoch:  15 Accuracy:  0.8491925 Training iteration:  15000 Training time:  649 s Training loss:  0.3632572 Learning rate 1.1998000000000423e-07\n",
      "Epoch:  16 Accuracy:  0.84831 Training iteration:  16000 Training time:  693 s Training loss:  0.3637448 Learning rate 2.0060020000000023e-05\n",
      "Epoch:  17 Accuracy:  0.8487775 Training iteration:  17000 Training time:  736 s Training loss:  0.36611688 Learning rate 4.004002e-05\n",
      "Epoch:  18 Accuracy:  0.84914 Training iteration:  18000 Training time:  779 s Training loss:  0.36488467 Learning rate 6.002002000000001e-05\n",
      "Epoch:  19 Accuracy:  0.84862 Training iteration:  19000 Training time:  822 s Training loss:  0.3656324 Learning rate 8.000002e-05\n",
      "Epoch:  20 Accuracy:  0.8487775 Training iteration:  20000 Training time:  865 s Training loss:  0.3640475 Learning rate 9.998002e-05\n",
      "Epoch:  21 Accuracy:  0.8507875 Training iteration:  21000 Training time:  909 s Training loss:  0.3632432 Learning rate 8.003998000000003e-05\n",
      "Model saved in file: checkpoints/test_model\n",
      "Epoch:  22 Accuracy:  0.8515025 Training iteration:  22000 Training time:  952 s Training loss:  0.3614631 Learning rate 6.005998000000001e-05\n",
      "Model saved in file: checkpoints/test_model\n",
      "Epoch:  23 Accuracy:  0.850795 Training iteration:  23000 Training time:  996 s Training loss:  0.36388892 Learning rate 4.0079979999999994e-05\n",
      "Epoch:  24 Accuracy:  0.850695 Training iteration:  24000 Training time:  1039 s Training loss:  0.3623561 Learning rate 2.0099979999999972e-05\n",
      "Epoch:  25 Accuracy:  0.85068 Training iteration:  25000 Training time:  1082 s Training loss:  0.362185 Learning rate 1.1998000000004489e-07\n",
      "Epoch:  26 Accuracy:  0.849285 Training iteration:  26000 Training time:  1125 s Training loss:  0.36036652 Learning rate 2.0060019999999982e-05\n",
      "Epoch:  27 Accuracy:  0.849965 Training iteration:  27000 Training time:  1169 s Training loss:  0.36372626 Learning rate 4.004002e-05\n",
      "Epoch:  28 Accuracy:  0.8490175 Training iteration:  28000 Training time:  1212 s Training loss:  0.36401817 Learning rate 6.002002000000001e-05\n",
      "Epoch:  29 Accuracy:  0.8499175 Training iteration:  29000 Training time:  1255 s Training loss:  0.3639024 Learning rate 8.000002000000004e-05\n",
      "Epoch:  30 Accuracy:  0.8502775 Training iteration:  30000 Training time:  1298 s Training loss:  0.3619758 Learning rate 9.998001999999996e-05\n",
      "Epoch:  31 Accuracy:  0.851025 Training iteration:  31000 Training time:  1340 s Training loss:  0.36101997 Learning rate 8.003998000000003e-05\n",
      "Epoch:  32 Accuracy:  0.8527625 Training iteration:  32000 Training time:  1383 s Training loss:  0.35929167 Learning rate 6.005998000000001e-05\n",
      "Model saved in file: checkpoints/test_model\n",
      "Epoch:  33 Accuracy:  0.8520625 Training iteration:  33000 Training time:  1426 s Training loss:  0.3572929 Learning rate 4.0079979999999994e-05\n",
      "Epoch:  34 Accuracy:  0.8530925 Training iteration:  34000 Training time:  1468 s Training loss:  0.3571297 Learning rate 2.0099979999999972e-05\n",
      "Model saved in file: checkpoints/test_model\n",
      "Epoch:  35 Accuracy:  0.852115 Training iteration:  35000 Training time:  1511 s Training loss:  0.3552367 Learning rate 1.1998000000004489e-07\n",
      "Epoch:  36 Accuracy:  0.853325 Training iteration:  36000 Training time:  1553 s Training loss:  0.35515594 Learning rate 2.0060019999999982e-05\n",
      "Model saved in file: checkpoints/test_model\n",
      "Epoch:  37 Accuracy:  0.852675 Training iteration:  37000 Training time:  1596 s Training loss:  0.35602564 Learning rate 4.004002e-05\n",
      "Epoch:  38 Accuracy:  0.852925 Training iteration:  38000 Training time:  1638 s Training loss:  0.35217026 Learning rate 6.002002000000001e-05\n",
      "Epoch:  39 Accuracy:  0.8551625 Training iteration:  39000 Training time:  1681 s Training loss:  0.3498301 Learning rate 8.000002000000004e-05\n",
      "Model saved in file: checkpoints/test_model\n",
      "Epoch:  40 Accuracy:  0.856505 Training iteration:  40000 Training time:  1725 s Training loss:  0.34773782 Learning rate 9.998001999999996e-05\n",
      "Model saved in file: checkpoints/test_model\n",
      "Epoch:  41 Accuracy:  0.8580325 Training iteration:  41000 Training time:  1768 s Training loss:  0.34012267 Learning rate 8.003998000000003e-05\n",
      "Model saved in file: checkpoints/test_model\n",
      "Epoch:  42 Accuracy:  0.85959 Training iteration:  42000 Training time:  1811 s Training loss:  0.3398928 Learning rate 6.005997999999992e-05\n",
      "Model saved in file: checkpoints/test_model\n",
      "Epoch:  43 Accuracy:  0.86003 Training iteration:  43000 Training time:  1854 s Training loss:  0.33525383 Learning rate 4.0079979999999994e-05\n",
      "Model saved in file: checkpoints/test_model\n",
      "Epoch:  44 Accuracy:  0.86267 Training iteration:  44000 Training time:  1897 s Training loss:  0.3322481 Learning rate 2.0099980000000067e-05\n",
      "Model saved in file: checkpoints/test_model\n",
      "Epoch:  45 Accuracy:  0.8622925 Training iteration:  45000 Training time:  1940 s Training loss:  0.33250406 Learning rate 1.1997999999995002e-07\n",
      "Epoch:  46 Accuracy:  0.862785 Training iteration:  46000 Training time:  1984 s Training loss:  0.33091217 Learning rate 2.0060019999999982e-05\n",
      "Model saved in file: checkpoints/test_model\n",
      "Epoch:  47 Accuracy:  0.8635025 Training iteration:  47000 Training time:  2027 s Training loss:  0.3297157 Learning rate 4.0040020000000086e-05\n",
      "Model saved in file: checkpoints/test_model\n",
      "Epoch:  48 Accuracy:  0.8646225 Training iteration:  48000 Training time:  2071 s Training loss:  0.32304305 Learning rate 6.002002000000001e-05\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model saved in file: checkpoints/test_model\n",
      "Epoch:  49 Accuracy:  0.8673425 Training iteration:  49000 Training time:  2115 s Training loss:  0.31870994 Learning rate 8.000001999999994e-05\n",
      "Model saved in file: checkpoints/test_model\n",
      "Epoch:  50 Accuracy:  0.8711225 Training iteration:  50000 Training time:  2159 s Training loss:  0.3075071 Learning rate 9.998002000000006e-05\n",
      "Model saved in file: checkpoints/test_model\n",
      "Epoch:  51 Accuracy:  0.87648 Training iteration:  51000 Training time:  2202 s Training loss:  0.29869553 Learning rate 8.003998000000003e-05\n",
      "Model saved in file: checkpoints/test_model\n",
      "Epoch:  52 Accuracy:  0.87977 Training iteration:  52000 Training time:  2246 s Training loss:  0.28989547 Learning rate 6.005997999999992e-05\n",
      "Model saved in file: checkpoints/test_model\n",
      "Epoch:  53 Accuracy:  0.8814075 Training iteration:  53000 Training time:  2290 s Training loss:  0.28205243 Learning rate 4.0079979999999994e-05\n",
      "Model saved in file: checkpoints/test_model\n",
      "Epoch:  54 Accuracy:  0.88561 Training iteration:  54000 Training time:  2334 s Training loss:  0.27662903 Learning rate 2.0099980000000067e-05\n",
      "Model saved in file: checkpoints/test_model\n",
      "Epoch:  55 Accuracy:  0.884755 Training iteration:  55000 Training time:  2377 s Training loss:  0.27195936 Learning rate 1.1997999999995002e-07\n",
      "Epoch:  56 Accuracy:  0.8846125 Training iteration:  56000 Training time:  2420 s Training loss:  0.2745338 Learning rate 2.0060019999999982e-05\n",
      "Epoch:  57 Accuracy:  0.886255 Training iteration:  57000 Training time:  2464 s Training loss:  0.268669 Learning rate 4.0040020000000086e-05\n",
      "Model saved in file: checkpoints/test_model\n",
      "Epoch:  58 Accuracy:  0.8879675 Training iteration:  58000 Training time:  2508 s Training loss:  0.26557505 Learning rate 6.002002000000001e-05\n",
      "Model saved in file: checkpoints/test_model\n",
      "Epoch:  59 Accuracy:  0.891825 Training iteration:  59000 Training time:  2551 s Training loss:  0.2553723 Learning rate 8.000001999999994e-05\n"
     ]
    },
    {
     "ename": "ResourceExhaustedError",
     "evalue": "checkpoints/test_model-59000.data-00000-of-00001.tempstate4085270137444843128; No space left on device\n\t [[Node: save/SaveV2 = SaveV2[dtypes=[DT_FLOAT, DT_FLOAT, DT_FLOAT, DT_FLOAT, DT_FLOAT, ..., DT_FLOAT, DT_FLOAT, DT_FLOAT, DT_FLOAT, DT_FLOAT], _device=\"/job:localhost/replica:0/task:0/device:CPU:0\"](_arg_save/Const_0_0, save/SaveV2/tensor_names, save/SaveV2/shape_and_slices, BatchNorm/beta/_31, BatchNorm/beta/Adam/_33, BatchNorm/beta/Adam_1/_35, BatchNorm/moving_mean/_37, BatchNorm/moving_variance/_39, beta1_power/_41, beta2_power/_43, chan_dec_h1/bias/_45, chan_dec_h1/bias/Adam/_47, chan_dec_h1/bias/Adam_1/_49, chan_dec_h1/kernel/_51, chan_dec_h1/kernel/Adam/_53, chan_dec_h1/kernel/Adam_1/_55, chan_dec_h2/bias/_57, chan_dec_h2/bias/Adam/_59, chan_dec_h2/bias/Adam_1/_61, chan_dec_h2/kernel/_63, chan_dec_h2/kernel/Adam/_65, chan_dec_h2/kernel/Adam_1/_67, chan_dec_h3/bias/_69, chan_dec_h3/bias/Adam/_71, chan_dec_h3/bias/Adam_1/_73, chan_dec_h3/kernel/_75, chan_dec_h3/kernel/Adam/_77, chan_dec_h3/kernel/Adam_1/_79, chan_dec_h4/bias/_81, chan_dec_h4/bias/Adam/_83, chan_dec_h4/bias/Adam_1/_85, chan_dec_h4/kernel/_87, chan_dec_h4/kernel/Adam/_89, chan_dec_h4/kernel/Adam_1/_91, chan_dec_h5/bias/_93, chan_dec_h5/bias/Adam/_95, chan_dec_h5/bias/Adam_1/_97, chan_dec_h5/kernel/_99, chan_dec_h5/kernel/Adam/_101, chan_dec_h5/kernel/Adam_1/_103, chan_dec_out/bias/_105, chan_dec_out/bias/Adam/_107, chan_dec_out/bias/Adam_1/_109, chan_dec_out/kernel/_111, chan_dec_out/kernel/Adam/_113, chan_dec_out/kernel/Adam_1/_115, chan_enc_h1/bias/_117, chan_enc_h1/bias/Adam/_119, chan_enc_h1/bias/Adam_1/_121, chan_enc_h1/kernel/_123, chan_enc_h1/kernel/Adam/_125, chan_enc_h1/kernel/Adam_1/_127, chan_enc_h2/bias/_129, chan_enc_h2/bias/Adam/_131, chan_enc_h2/bias/Adam_1/_133, chan_enc_h2/kernel/_135, chan_enc_h2/kernel/Adam/_137, chan_enc_h2/kernel/Adam_1/_139, chan_enc_h3/bias/_141, chan_enc_h3/bias/Adam/_143, chan_enc_h3/bias/Adam_1/_145, chan_enc_h3/kernel/_147, chan_enc_h3/kernel/Adam/_149, chan_enc_h3/kernel/Adam_1/_151, chan_enc_h4/bias/_153, chan_enc_h4/bias/Adam/_155, chan_enc_h4/bias/Adam_1/_157, chan_enc_h4/kernel/_159, chan_enc_h4/kernel/Adam/_161, chan_enc_h4/kernel/Adam_1/_163, chan_enc_out/bias/_165, chan_enc_out/bias/Adam/_167, chan_enc_out/bias/Adam_1/_169, chan_enc_out/kernel/_171, chan_enc_out/kernel/Adam/_173, chan_enc_out/kernel/Adam_1/_175)]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info.\n\n\nCaused by op 'save/SaveV2', defined at:\n  File \"/usr/lib/python3.6/runpy.py\", line 193, in _run_module_as_main\n    \"__main__\", mod_spec)\n  File \"/usr/lib/python3.6/runpy.py\", line 85, in _run_code\n    exec(code, run_globals)\n  File \"/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py\", line 16, in <module>\n    app.launch_new_instance()\n  File \"/usr/local/lib/python3.6/dist-packages/traitlets/config/application.py\", line 658, in launch_instance\n    app.start()\n  File \"/usr/local/lib/python3.6/dist-packages/ipykernel/kernelapp.py\", line 486, in start\n    self.io_loop.start()\n  File \"/usr/local/lib/python3.6/dist-packages/tornado/platform/asyncio.py\", line 112, in start\n    self.asyncio_loop.run_forever()\n  File \"/usr/lib/python3.6/asyncio/base_events.py\", line 422, in run_forever\n    self._run_once()\n  File \"/usr/lib/python3.6/asyncio/base_events.py\", line 1432, in _run_once\n    handle._run()\n  File \"/usr/lib/python3.6/asyncio/events.py\", line 145, in _run\n    self._callback(*self._args)\n  File \"/usr/local/lib/python3.6/dist-packages/tornado/platform/asyncio.py\", line 102, in _handle_events\n    handler_func(fileobj, events)\n  File \"/usr/local/lib/python3.6/dist-packages/tornado/stack_context.py\", line 276, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"/usr/local/lib/python3.6/dist-packages/zmq/eventloop/zmqstream.py\", line 450, in _handle_events\n    self._handle_recv()\n  File \"/usr/local/lib/python3.6/dist-packages/zmq/eventloop/zmqstream.py\", line 480, in _handle_recv\n    self._run_callback(callback, msg)\n  File \"/usr/local/lib/python3.6/dist-packages/zmq/eventloop/zmqstream.py\", line 432, in _run_callback\n    callback(*args, **kwargs)\n  File \"/usr/local/lib/python3.6/dist-packages/tornado/stack_context.py\", line 276, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"/usr/local/lib/python3.6/dist-packages/ipykernel/kernelbase.py\", line 283, in dispatcher\n    return self.dispatch_shell(stream, msg)\n  File \"/usr/local/lib/python3.6/dist-packages/ipykernel/kernelbase.py\", line 233, in dispatch_shell\n    handler(stream, idents, msg)\n  File \"/usr/local/lib/python3.6/dist-packages/ipykernel/kernelbase.py\", line 399, in execute_request\n    user_expressions, allow_stdin)\n  File \"/usr/local/lib/python3.6/dist-packages/ipykernel/ipkernel.py\", line 208, in do_execute\n    res = shell.run_cell(code, store_history=store_history, silent=silent)\n  File \"/usr/local/lib/python3.6/dist-packages/ipykernel/zmqshell.py\", line 537, in run_cell\n    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n  File \"/usr/local/lib/python3.6/dist-packages/IPython/core/interactiveshell.py\", line 2728, in run_cell\n    interactivity=interactivity, compiler=compiler, result=result)\n  File \"/usr/local/lib/python3.6/dist-packages/IPython/core/interactiveshell.py\", line 2850, in run_ast_nodes\n    if self.run_code(code, result):\n  File \"/usr/local/lib/python3.6/dist-packages/IPython/core/interactiveshell.py\", line 2910, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"<ipython-input-19-f71af896bb25>\", line 34, in <module>\n    train_accs, test_acc, learn_rates = train_model(chan_enc_config, num_train, num_test)\n  File \"<ipython-input-11-e5bc2afdffe2>\", line 3, in train_model\n    channel_sys = ChannelSystem(chan_enc_config, train_size,test_size)\n  File \"<ipython-input-16-c0bed315d201>\", line 64, in __init__\n    self.saver = tf.train.Saver(max_to_keep = self.config.save_max)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py\", line 1338, in __init__\n    self.build()\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py\", line 1347, in build\n    self._build(self._filename, build_save=True, build_restore=True)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py\", line 1384, in _build\n    build_save=build_save, build_restore=build_restore)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py\", line 832, in _build_internal\n    save_tensor = self._AddSaveOps(filename_tensor, saveables)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py\", line 350, in _AddSaveOps\n    save = self.save_op(filename_tensor, saveables)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py\", line 266, in save_op\n    tensors)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/gen_io_ops.py\", line 1687, in save_v2\n    shape_and_slices=shape_and_slices, tensors=tensors, name=name)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/op_def_library.py\", line 787, in _apply_op_helper\n    op_def=op_def)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/ops.py\", line 3392, in create_op\n    op_def=op_def)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/ops.py\", line 1718, in __init__\n    self._traceback = self._graph._extract_stack()  # pylint: disable=protected-access\n\nResourceExhaustedError (see above for traceback): checkpoints/test_model-59000.data-00000-of-00001.tempstate4085270137444843128; No space left on device\n\t [[Node: save/SaveV2 = SaveV2[dtypes=[DT_FLOAT, DT_FLOAT, DT_FLOAT, DT_FLOAT, DT_FLOAT, ..., DT_FLOAT, DT_FLOAT, DT_FLOAT, DT_FLOAT, DT_FLOAT], _device=\"/job:localhost/replica:0/task:0/device:CPU:0\"](_arg_save/Const_0_0, save/SaveV2/tensor_names, save/SaveV2/shape_and_slices, BatchNorm/beta/_31, BatchNorm/beta/Adam/_33, BatchNorm/beta/Adam_1/_35, BatchNorm/moving_mean/_37, BatchNorm/moving_variance/_39, beta1_power/_41, beta2_power/_43, chan_dec_h1/bias/_45, chan_dec_h1/bias/Adam/_47, chan_dec_h1/bias/Adam_1/_49, chan_dec_h1/kernel/_51, chan_dec_h1/kernel/Adam/_53, chan_dec_h1/kernel/Adam_1/_55, chan_dec_h2/bias/_57, chan_dec_h2/bias/Adam/_59, chan_dec_h2/bias/Adam_1/_61, chan_dec_h2/kernel/_63, chan_dec_h2/kernel/Adam/_65, chan_dec_h2/kernel/Adam_1/_67, chan_dec_h3/bias/_69, chan_dec_h3/bias/Adam/_71, chan_dec_h3/bias/Adam_1/_73, chan_dec_h3/kernel/_75, chan_dec_h3/kernel/Adam/_77, chan_dec_h3/kernel/Adam_1/_79, chan_dec_h4/bias/_81, chan_dec_h4/bias/Adam/_83, chan_dec_h4/bias/Adam_1/_85, chan_dec_h4/kernel/_87, chan_dec_h4/kernel/Adam/_89, chan_dec_h4/kernel/Adam_1/_91, chan_dec_h5/bias/_93, chan_dec_h5/bias/Adam/_95, chan_dec_h5/bias/Adam_1/_97, chan_dec_h5/kernel/_99, chan_dec_h5/kernel/Adam/_101, chan_dec_h5/kernel/Adam_1/_103, chan_dec_out/bias/_105, chan_dec_out/bias/Adam/_107, chan_dec_out/bias/Adam_1/_109, chan_dec_out/kernel/_111, chan_dec_out/kernel/Adam/_113, chan_dec_out/kernel/Adam_1/_115, chan_enc_h1/bias/_117, chan_enc_h1/bias/Adam/_119, chan_enc_h1/bias/Adam_1/_121, chan_enc_h1/kernel/_123, chan_enc_h1/kernel/Adam/_125, chan_enc_h1/kernel/Adam_1/_127, chan_enc_h2/bias/_129, chan_enc_h2/bias/Adam/_131, chan_enc_h2/bias/Adam_1/_133, chan_enc_h2/kernel/_135, chan_enc_h2/kernel/Adam/_137, chan_enc_h2/kernel/Adam_1/_139, chan_enc_h3/bias/_141, chan_enc_h3/bias/Adam/_143, chan_enc_h3/bias/Adam_1/_145, chan_enc_h3/kernel/_147, chan_enc_h3/kernel/Adam/_149, chan_enc_h3/kernel/Adam_1/_151, chan_enc_h4/bias/_153, chan_enc_h4/bias/Adam/_155, chan_enc_h4/bias/Adam_1/_157, chan_enc_h4/kernel/_159, chan_enc_h4/kernel/Adam/_161, chan_enc_h4/kernel/Adam_1/_163, chan_enc_out/bias/_165, chan_enc_out/bias/Adam/_167, chan_enc_out/bias/Adam_1/_169, chan_enc_out/kernel/_171, chan_enc_out/kernel/Adam/_173, chan_enc_out/kernel/Adam_1/_175)]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info.\n\n",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mResourceExhaustedError\u001b[0m                    Traceback (most recent call last)",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1321\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1322\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1323\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1306\u001b[0m       return self._call_tf_sessionrun(\n\u001b[0;32m-> 1307\u001b[0;31m           options, feed_dict, fetch_list, target_list, run_metadata)\n\u001b[0m\u001b[1;32m   1308\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_call_tf_sessionrun\u001b[0;34m(self, options, feed_dict, fetch_list, target_list, run_metadata)\u001b[0m\n\u001b[1;32m   1408\u001b[0m           \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1409\u001b[0;31m           run_metadata)\n\u001b[0m\u001b[1;32m   1410\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mResourceExhaustedError\u001b[0m: checkpoints/test_model-59000.data-00000-of-00001.tempstate4085270137444843128; No space left on device\n\t [[Node: save/SaveV2 = SaveV2[dtypes=[DT_FLOAT, DT_FLOAT, DT_FLOAT, DT_FLOAT, DT_FLOAT, ..., DT_FLOAT, DT_FLOAT, DT_FLOAT, DT_FLOAT, DT_FLOAT], _device=\"/job:localhost/replica:0/task:0/device:CPU:0\"](_arg_save/Const_0_0, save/SaveV2/tensor_names, save/SaveV2/shape_and_slices, BatchNorm/beta/_31, BatchNorm/beta/Adam/_33, BatchNorm/beta/Adam_1/_35, BatchNorm/moving_mean/_37, BatchNorm/moving_variance/_39, beta1_power/_41, beta2_power/_43, chan_dec_h1/bias/_45, chan_dec_h1/bias/Adam/_47, chan_dec_h1/bias/Adam_1/_49, chan_dec_h1/kernel/_51, chan_dec_h1/kernel/Adam/_53, chan_dec_h1/kernel/Adam_1/_55, chan_dec_h2/bias/_57, chan_dec_h2/bias/Adam/_59, chan_dec_h2/bias/Adam_1/_61, chan_dec_h2/kernel/_63, chan_dec_h2/kernel/Adam/_65, chan_dec_h2/kernel/Adam_1/_67, chan_dec_h3/bias/_69, chan_dec_h3/bias/Adam/_71, chan_dec_h3/bias/Adam_1/_73, chan_dec_h3/kernel/_75, chan_dec_h3/kernel/Adam/_77, chan_dec_h3/kernel/Adam_1/_79, chan_dec_h4/bias/_81, chan_dec_h4/bias/Adam/_83, chan_dec_h4/bias/Adam_1/_85, chan_dec_h4/kernel/_87, chan_dec_h4/kernel/Adam/_89, chan_dec_h4/kernel/Adam_1/_91, chan_dec_h5/bias/_93, chan_dec_h5/bias/Adam/_95, chan_dec_h5/bias/Adam_1/_97, chan_dec_h5/kernel/_99, chan_dec_h5/kernel/Adam/_101, chan_dec_h5/kernel/Adam_1/_103, chan_dec_out/bias/_105, chan_dec_out/bias/Adam/_107, chan_dec_out/bias/Adam_1/_109, chan_dec_out/kernel/_111, chan_dec_out/kernel/Adam/_113, chan_dec_out/kernel/Adam_1/_115, chan_enc_h1/bias/_117, chan_enc_h1/bias/Adam/_119, chan_enc_h1/bias/Adam_1/_121, chan_enc_h1/kernel/_123, chan_enc_h1/kernel/Adam/_125, chan_enc_h1/kernel/Adam_1/_127, chan_enc_h2/bias/_129, chan_enc_h2/bias/Adam/_131, chan_enc_h2/bias/Adam_1/_133, chan_enc_h2/kernel/_135, chan_enc_h2/kernel/Adam/_137, chan_enc_h2/kernel/Adam_1/_139, chan_enc_h3/bias/_141, chan_enc_h3/bias/Adam/_143, chan_enc_h3/bias/Adam_1/_145, chan_enc_h3/kernel/_147, chan_enc_h3/kernel/Adam/_149, chan_enc_h3/kernel/Adam_1/_151, chan_enc_h4/bias/_153, chan_enc_h4/bias/Adam/_155, chan_enc_h4/bias/Adam_1/_157, chan_enc_h4/kernel/_159, chan_enc_h4/kernel/Adam/_161, chan_enc_h4/kernel/Adam_1/_163, chan_enc_out/bias/_165, chan_enc_out/bias/Adam/_167, chan_enc_out/bias/Adam_1/_169, chan_enc_out/kernel/_171, chan_enc_out/kernel/Adam/_173, chan_enc_out/kernel/Adam_1/_175)]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info.\n",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mResourceExhaustedError\u001b[0m                    Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-19-f71af896bb25>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     32\u001b[0m                         save_max = 10)\n\u001b[1;32m     33\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 34\u001b[0;31m \u001b[0mtrain_accs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest_acc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlearn_rates\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchan_enc_config\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnum_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnum_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-11-e5bc2afdffe2>\u001b[0m in \u001b[0;36mtrain_model\u001b[0;34m(config, train_size, test_size)\u001b[0m\n\u001b[1;32m      3\u001b[0m     \u001b[0mchannel_sys\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mChannelSystem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchan_enc_config\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_size\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mtest_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSession\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0msess\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m         \u001b[0mtrain_acc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest_acc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlearn_rates\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mchannel_sys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_and_test\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msess\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mtrain_acc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest_acc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlearn_rates\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-16-c0bed315d201>\u001b[0m in \u001b[0;36mtrain_and_test\u001b[0;34m(self, sess, grammar)\u001b[0m\n\u001b[1;32m    165\u001b[0m                     \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtraining_counter\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miter_save_rate\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    166\u001b[0m                         \u001b[0;32mif\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maccuracies\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m1\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0maccuracies\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mamax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maccuracies\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 167\u001b[0;31m                             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msaver\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msess\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel_save_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mglobal_step\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtraining_counter\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    168\u001b[0m                             \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Model saved in file: %s\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel_save_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    169\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(self, sess, save_path, global_step, latest_filename, meta_graph_suffix, write_meta_graph, write_state, strip_default_attrs)\u001b[0m\n\u001b[1;32m   1701\u001b[0m           model_checkpoint_path = sess.run(\n\u001b[1;32m   1702\u001b[0m               \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msaver_def\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave_tensor_name\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1703\u001b[0;31m               {self.saver_def.filename_tensor_name: checkpoint_file})\n\u001b[0m\u001b[1;32m   1704\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1705\u001b[0m         \u001b[0mmodel_checkpoint_path\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_str\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel_checkpoint_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    898\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    899\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 900\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    901\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    902\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1133\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1134\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1135\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1136\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1137\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1314\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1315\u001b[0m       return self._do_call(_run_fn, feeds, fetches, targets, options,\n\u001b[0;32m-> 1316\u001b[0;31m                            run_metadata)\n\u001b[0m\u001b[1;32m   1317\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1318\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1333\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1334\u001b[0m           \u001b[0;32mpass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1335\u001b[0;31m       \u001b[0;32mraise\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnode_def\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1336\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1337\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mResourceExhaustedError\u001b[0m: checkpoints/test_model-59000.data-00000-of-00001.tempstate4085270137444843128; No space left on device\n\t [[Node: save/SaveV2 = SaveV2[dtypes=[DT_FLOAT, DT_FLOAT, DT_FLOAT, DT_FLOAT, DT_FLOAT, ..., DT_FLOAT, DT_FLOAT, DT_FLOAT, DT_FLOAT, DT_FLOAT], _device=\"/job:localhost/replica:0/task:0/device:CPU:0\"](_arg_save/Const_0_0, save/SaveV2/tensor_names, save/SaveV2/shape_and_slices, BatchNorm/beta/_31, BatchNorm/beta/Adam/_33, BatchNorm/beta/Adam_1/_35, BatchNorm/moving_mean/_37, BatchNorm/moving_variance/_39, beta1_power/_41, beta2_power/_43, chan_dec_h1/bias/_45, chan_dec_h1/bias/Adam/_47, chan_dec_h1/bias/Adam_1/_49, chan_dec_h1/kernel/_51, chan_dec_h1/kernel/Adam/_53, chan_dec_h1/kernel/Adam_1/_55, chan_dec_h2/bias/_57, chan_dec_h2/bias/Adam/_59, chan_dec_h2/bias/Adam_1/_61, chan_dec_h2/kernel/_63, chan_dec_h2/kernel/Adam/_65, chan_dec_h2/kernel/Adam_1/_67, chan_dec_h3/bias/_69, chan_dec_h3/bias/Adam/_71, chan_dec_h3/bias/Adam_1/_73, chan_dec_h3/kernel/_75, chan_dec_h3/kernel/Adam/_77, chan_dec_h3/kernel/Adam_1/_79, chan_dec_h4/bias/_81, chan_dec_h4/bias/Adam/_83, chan_dec_h4/bias/Adam_1/_85, chan_dec_h4/kernel/_87, chan_dec_h4/kernel/Adam/_89, chan_dec_h4/kernel/Adam_1/_91, chan_dec_h5/bias/_93, chan_dec_h5/bias/Adam/_95, chan_dec_h5/bias/Adam_1/_97, chan_dec_h5/kernel/_99, chan_dec_h5/kernel/Adam/_101, chan_dec_h5/kernel/Adam_1/_103, chan_dec_out/bias/_105, chan_dec_out/bias/Adam/_107, chan_dec_out/bias/Adam_1/_109, chan_dec_out/kernel/_111, chan_dec_out/kernel/Adam/_113, chan_dec_out/kernel/Adam_1/_115, chan_enc_h1/bias/_117, chan_enc_h1/bias/Adam/_119, chan_enc_h1/bias/Adam_1/_121, chan_enc_h1/kernel/_123, chan_enc_h1/kernel/Adam/_125, chan_enc_h1/kernel/Adam_1/_127, chan_enc_h2/bias/_129, chan_enc_h2/bias/Adam/_131, chan_enc_h2/bias/Adam_1/_133, chan_enc_h2/kernel/_135, chan_enc_h2/kernel/Adam/_137, chan_enc_h2/kernel/Adam_1/_139, chan_enc_h3/bias/_141, chan_enc_h3/bias/Adam/_143, chan_enc_h3/bias/Adam_1/_145, chan_enc_h3/kernel/_147, chan_enc_h3/kernel/Adam/_149, chan_enc_h3/kernel/Adam_1/_151, chan_enc_h4/bias/_153, chan_enc_h4/bias/Adam/_155, chan_enc_h4/bias/Adam_1/_157, chan_enc_h4/kernel/_159, chan_enc_h4/kernel/Adam/_161, chan_enc_h4/kernel/Adam_1/_163, chan_enc_out/bias/_165, chan_enc_out/bias/Adam/_167, chan_enc_out/bias/Adam_1/_169, chan_enc_out/kernel/_171, chan_enc_out/kernel/Adam/_173, chan_enc_out/kernel/Adam_1/_175)]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info.\n\n\nCaused by op 'save/SaveV2', defined at:\n  File \"/usr/lib/python3.6/runpy.py\", line 193, in _run_module_as_main\n    \"__main__\", mod_spec)\n  File \"/usr/lib/python3.6/runpy.py\", line 85, in _run_code\n    exec(code, run_globals)\n  File \"/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py\", line 16, in <module>\n    app.launch_new_instance()\n  File \"/usr/local/lib/python3.6/dist-packages/traitlets/config/application.py\", line 658, in launch_instance\n    app.start()\n  File \"/usr/local/lib/python3.6/dist-packages/ipykernel/kernelapp.py\", line 486, in start\n    self.io_loop.start()\n  File \"/usr/local/lib/python3.6/dist-packages/tornado/platform/asyncio.py\", line 112, in start\n    self.asyncio_loop.run_forever()\n  File \"/usr/lib/python3.6/asyncio/base_events.py\", line 422, in run_forever\n    self._run_once()\n  File \"/usr/lib/python3.6/asyncio/base_events.py\", line 1432, in _run_once\n    handle._run()\n  File \"/usr/lib/python3.6/asyncio/events.py\", line 145, in _run\n    self._callback(*self._args)\n  File \"/usr/local/lib/python3.6/dist-packages/tornado/platform/asyncio.py\", line 102, in _handle_events\n    handler_func(fileobj, events)\n  File \"/usr/local/lib/python3.6/dist-packages/tornado/stack_context.py\", line 276, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"/usr/local/lib/python3.6/dist-packages/zmq/eventloop/zmqstream.py\", line 450, in _handle_events\n    self._handle_recv()\n  File \"/usr/local/lib/python3.6/dist-packages/zmq/eventloop/zmqstream.py\", line 480, in _handle_recv\n    self._run_callback(callback, msg)\n  File \"/usr/local/lib/python3.6/dist-packages/zmq/eventloop/zmqstream.py\", line 432, in _run_callback\n    callback(*args, **kwargs)\n  File \"/usr/local/lib/python3.6/dist-packages/tornado/stack_context.py\", line 276, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"/usr/local/lib/python3.6/dist-packages/ipykernel/kernelbase.py\", line 283, in dispatcher\n    return self.dispatch_shell(stream, msg)\n  File \"/usr/local/lib/python3.6/dist-packages/ipykernel/kernelbase.py\", line 233, in dispatch_shell\n    handler(stream, idents, msg)\n  File \"/usr/local/lib/python3.6/dist-packages/ipykernel/kernelbase.py\", line 399, in execute_request\n    user_expressions, allow_stdin)\n  File \"/usr/local/lib/python3.6/dist-packages/ipykernel/ipkernel.py\", line 208, in do_execute\n    res = shell.run_cell(code, store_history=store_history, silent=silent)\n  File \"/usr/local/lib/python3.6/dist-packages/ipykernel/zmqshell.py\", line 537, in run_cell\n    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n  File \"/usr/local/lib/python3.6/dist-packages/IPython/core/interactiveshell.py\", line 2728, in run_cell\n    interactivity=interactivity, compiler=compiler, result=result)\n  File \"/usr/local/lib/python3.6/dist-packages/IPython/core/interactiveshell.py\", line 2850, in run_ast_nodes\n    if self.run_code(code, result):\n  File \"/usr/local/lib/python3.6/dist-packages/IPython/core/interactiveshell.py\", line 2910, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"<ipython-input-19-f71af896bb25>\", line 34, in <module>\n    train_accs, test_acc, learn_rates = train_model(chan_enc_config, num_train, num_test)\n  File \"<ipython-input-11-e5bc2afdffe2>\", line 3, in train_model\n    channel_sys = ChannelSystem(chan_enc_config, train_size,test_size)\n  File \"<ipython-input-16-c0bed315d201>\", line 64, in __init__\n    self.saver = tf.train.Saver(max_to_keep = self.config.save_max)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py\", line 1338, in __init__\n    self.build()\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py\", line 1347, in build\n    self._build(self._filename, build_save=True, build_restore=True)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py\", line 1384, in _build\n    build_save=build_save, build_restore=build_restore)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py\", line 832, in _build_internal\n    save_tensor = self._AddSaveOps(filename_tensor, saveables)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py\", line 350, in _AddSaveOps\n    save = self.save_op(filename_tensor, saveables)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py\", line 266, in save_op\n    tensors)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/gen_io_ops.py\", line 1687, in save_v2\n    shape_and_slices=shape_and_slices, tensors=tensors, name=name)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/op_def_library.py\", line 787, in _apply_op_helper\n    op_def=op_def)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/ops.py\", line 3392, in create_op\n    op_def=op_def)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/ops.py\", line 1718, in __init__\n    self._traceback = self._graph._extract_stack()  # pylint: disable=protected-access\n\nResourceExhaustedError (see above for traceback): checkpoints/test_model-59000.data-00000-of-00001.tempstate4085270137444843128; No space left on device\n\t [[Node: save/SaveV2 = SaveV2[dtypes=[DT_FLOAT, DT_FLOAT, DT_FLOAT, DT_FLOAT, DT_FLOAT, ..., DT_FLOAT, DT_FLOAT, DT_FLOAT, DT_FLOAT, DT_FLOAT], _device=\"/job:localhost/replica:0/task:0/device:CPU:0\"](_arg_save/Const_0_0, save/SaveV2/tensor_names, save/SaveV2/shape_and_slices, BatchNorm/beta/_31, BatchNorm/beta/Adam/_33, BatchNorm/beta/Adam_1/_35, BatchNorm/moving_mean/_37, BatchNorm/moving_variance/_39, beta1_power/_41, beta2_power/_43, chan_dec_h1/bias/_45, chan_dec_h1/bias/Adam/_47, chan_dec_h1/bias/Adam_1/_49, chan_dec_h1/kernel/_51, chan_dec_h1/kernel/Adam/_53, chan_dec_h1/kernel/Adam_1/_55, chan_dec_h2/bias/_57, chan_dec_h2/bias/Adam/_59, chan_dec_h2/bias/Adam_1/_61, chan_dec_h2/kernel/_63, chan_dec_h2/kernel/Adam/_65, chan_dec_h2/kernel/Adam_1/_67, chan_dec_h3/bias/_69, chan_dec_h3/bias/Adam/_71, chan_dec_h3/bias/Adam_1/_73, chan_dec_h3/kernel/_75, chan_dec_h3/kernel/Adam/_77, chan_dec_h3/kernel/Adam_1/_79, chan_dec_h4/bias/_81, chan_dec_h4/bias/Adam/_83, chan_dec_h4/bias/Adam_1/_85, chan_dec_h4/kernel/_87, chan_dec_h4/kernel/Adam/_89, chan_dec_h4/kernel/Adam_1/_91, chan_dec_h5/bias/_93, chan_dec_h5/bias/Adam/_95, chan_dec_h5/bias/Adam_1/_97, chan_dec_h5/kernel/_99, chan_dec_h5/kernel/Adam/_101, chan_dec_h5/kernel/Adam_1/_103, chan_dec_out/bias/_105, chan_dec_out/bias/Adam/_107, chan_dec_out/bias/Adam_1/_109, chan_dec_out/kernel/_111, chan_dec_out/kernel/Adam/_113, chan_dec_out/kernel/Adam_1/_115, chan_enc_h1/bias/_117, chan_enc_h1/bias/Adam/_119, chan_enc_h1/bias/Adam_1/_121, chan_enc_h1/kernel/_123, chan_enc_h1/kernel/Adam/_125, chan_enc_h1/kernel/Adam_1/_127, chan_enc_h2/bias/_129, chan_enc_h2/bias/Adam/_131, chan_enc_h2/bias/Adam_1/_133, chan_enc_h2/kernel/_135, chan_enc_h2/kernel/Adam/_137, chan_enc_h2/kernel/Adam_1/_139, chan_enc_h3/bias/_141, chan_enc_h3/bias/Adam/_143, chan_enc_h3/bias/Adam_1/_145, chan_enc_h3/kernel/_147, chan_enc_h3/kernel/Adam/_149, chan_enc_h3/kernel/Adam_1/_151, chan_enc_h4/bias/_153, chan_enc_h4/bias/Adam/_155, chan_enc_h4/bias/Adam_1/_157, chan_enc_h4/kernel/_159, chan_enc_h4/kernel/Adam/_161, chan_enc_h4/kernel/Adam_1/_163, chan_enc_out/bias/_165, chan_enc_out/bias/Adam/_167, chan_enc_out/bias/Adam_1/_169, chan_enc_out/kernel/_171, chan_enc_out/kernel/Adam/_173, chan_enc_out/kernel/Adam_1/_175)]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info.\n\n"
     ]
    }
   ],
   "source": [
    "chan_enc_layers = [4096, 2048, 1024, 512]\n",
    "chan_dec_layers = [4096, 2048, 1600, 1024, 512]\n",
    "\n",
    "chan_keep_prob = 0.925\n",
    "channel = {'type':'erasure','chan_param':chan_keep_prob}\n",
    "\n",
    "chan_enc_in_len = 400\n",
    "code_rate = 0.90\n",
    "num_tx_bits = np.ceil(chan_enc_in_len / code_rate)\n",
    "\n",
    "num_epochs = 150\n",
    "num_train = int(1e6)\n",
    "num_test = int(1e6)\n",
    "batch_size = 1000\n",
    "max_batch_in_epoch = int(1 * 1e3)\n",
    "learn_rate = 0.0001 \n",
    "model_save_path = \"checkpoints/test_model\"\n",
    "iter_save_rate = max_batch_in_epoch\n",
    "\n",
    "chan_enc_config = Config(chan_enc_layers,\n",
    "                        chan_dec_layers,\n",
    "                        num_tx_bits,\n",
    "                        chan_enc_in_len, #==chan_dec_out_len\n",
    "                        channel = channel,\n",
    "                        num_epochs = num_epochs,\n",
    "                        learn_rate = learn_rate,\n",
    "                        \n",
    "                        batch_size = batch_size,\n",
    "                        max_batch_in_epoch = max_batch_in_epoch, \n",
    "                        iter_save_rate = iter_save_rate,\n",
    "                        model_save_path = model_save_path,\n",
    "                        save_max = 10)\n",
    "\n",
    "train_accs, test_acc, learn_rates = train_model(chan_enc_config, num_train, num_test)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.964\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZAAAAD8CAYAAABZ/vJZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvFvnyVgAAIABJREFUeJztvXuQY9l93/c5ePYDuJjpNzAzuzM7j+0BbJVEj1lSyeWSSVVxSSlcVkw6yyonTMKIlYh07KzjcFlKsRQ6rPLKlayiMmmFMhVRiq3lauNEY2dFRjEZJVHMJYeiSBF3enZ6Hrs7c9GP6enBRaO7gQZw8se9txvT00Cj0QDuA+dTNTXoi4tzT5++uL9zvt/fOUdIKVEoFAqF4qiE3K6AQqFQKPyJCiAKhUKh6AoVQBQKhULRFSqAKBQKhaIrVABRKBQKRVeoAKJQKBSKrlABRKFQKBRdoQKIQqFQKLpCBRCFQqFQdEXE7Qr0k6mpKXn27Fm3q6FQKBS+4vvf//4DKeX0YecFOoCcPXuWa9euuV0NhUKh8BVCiLc7OU9JWAqFQqHoChVAFAqFQtEVKoAoFAqFoitUAFEoFApFV6gAolAoFIqu6CiACCGeE0LcEEIsCiFeOuD9uBDi6/b7bwohzja99zn7+A0hxAcOK1MI8Rn7mBRCTDUdF0KI37Df+5EQ4j3d/tIKhUKhOD6HBhAhRBj4EvBBIAt8XAiR3XfaJ4F1KeUF4BXgZfuzWeAFIAc8B3xZCBE+pMw/BX4e2J9G9kHgov3vU8A/PdqvqlAoFIpe0skI5L3AopTytpSyCrwKPL/vnOeBr9mvXwfeL4QQ9vFXpZQVKeUdYNEur2WZUsofSCnvHlCP54HflRbfAU4IIdJH+WU75drdh/yjP1pAbfd7OH9xr8i1uw/droYv+GN9mXcfbrpdDc9Tb0h+/7vvUKnV3a6K51ktVfhXPzRcu34nAeQU8G7Tz/fsYweeI6WsAUVgss1nOymzm3oghPiUEOKaEOLa6urqIUUeTN4w+c0/ucWyWenq88PEv/NP/l8++pv/1u1qeJ6tap1f+t1rfEy11aH8qx8afO5f/gVf+ZPbblfF8/zyP/8+f+f3f8Cyue3K9QNnokspvyKlvCKlvDI9fehM/APJZjQA9EKxl1ULHPXG3ghto1JzsSbe58ZyCYAll77ofuLeujVKu/9oy+WaeB/dMB/7f9B0EkDuA2eafj5tHzvwHCFEBEgBa20+20mZ3dSjJ1xO2wHEpT+KX7i7Vt59vVBQbdWOvLHXGanVGy7WxPvcWN4AVADpBKcPp7v0/eskgHwPuCiEOCeEiGGZ4lf3nXMV+IT9+qPAt6RlIFwFXrCztM5hGeDf7bDM/VwF/gM7G+ungaKUstBB/Y9MIh7h7OQYeRVA2tIcYFVbtae5rW6tltucqdDtYJs3TOVDtmGltM3WjuUTNXdQBsmhAcT2ND4DfBO4DrwmpcwLIb4ghPiwfdpXgUkhxCLwIvCS/dk88BqgA98APi2lrLcqE0AI8Z8LIe5hjTB+JIT4Z/Y13gBuYxnxvwX88rF/+zZkM5prUd0v6AWTaFiQHImo0doh6AWTyfGY/VpJo63YrNa4/aDMxHiMh+Wq8iHbcL1gyaIT4zFPS1hIKd+QUl6SUp6XUn7RPvZ5KeVV+/W2lPJjUsoLUsr3SilvN332i/bnnpVS/lG7Mu3jvyGlPC2ljEgpM1LK/8Q+LqWUn7bP/8tSyr4us5vLpHh7bRNze6efl/E1ecPk4kySnzxzQgXbNtQbkoVCiV/8iTTxSEgF2zYsLJWQEj72V04DKti2wxl1/M33nOLu2qYrPmTgTPRekbV9kAU7yiueRDdMshmNbFrjxlKJHaXtH8idB2W2dur85dMnmJ9LKrmvDU5w/XffYwWQ/H3VVq3QDZPTJ0f5mfOTAFx3oROnAkgLck4mlkvaotdZMbd5sFEhl9HIZjSq9Qa3VjfcrpYncXqKTlvpBaXttyJvmKRGo1yaTXB2ckyNbNugG6Z1T6VTuz8PGhVAWjCdjDOViKneYgvy9hc7m9Z2g63qLR6MXjCJhUNcmEmQTWs82tzBKKp03oPQCybZtIYQglwmpb5/LShXatxZK5NNp5jV4kyOx1wx0lUAaYEQgstpZaS3wuntXM5onJtKMBINqbZqgW6YXJpLEA2HyGbc6y16nVq9wULB3O2QZDMa7zxUPuRBOF5RLmMFW7eSflQAaUMuk+Kt5RLVmtL296MbJk9NjKGNRAmHBPNzmnooHoCU0vKKbE9tfi6JECqAHMSdB2UqtcbuRF7nf+VDPokjre+2VVrjraWNgfuQKoC0IZvR2KlLFleUtr8fR2pwyGY08kZRafv7WClVWCtXd9tqPB7h3NS4a3n7XsbpQTsPxZzdZqqtnkQvmJwYi5JOjQDs+pCDflapANKGXSNdSTOPsVGpcedBebd9wOoBmds1NXt4H7sG+qnU7rGskkYPJG+YxCIhzk8ngD0fUo3WniRvG+jWmrXNST+DbSsVQNpwdnKc0WhY9YD2sbCvpwh7N7AyPR/H+ULPzyV3j2UzGvfWtyhuKm2/Gd0weXY2STRsPZYsbV8Z6fup1RssLJUeUwAcH3LQbaUCSBvCIcF8Oql6QPtwes+5zF6ven5OI6S0/SfQCyZnJ8dIjkR3jzntpkYhe0gp0ZsMdIdsWuPmivIhm7m1WqZaazz2/dv1IQc88VIFkEPIqbz9J8jfN5kYjzGrxXePjcbCPDOdUA/FfeTtyZbNOD1H1VZ7LJnbPCxXn2irnPIhn8AJEk/cVxkrkWWQzyoVQA4hm05R2q5xb11p+w7NufrNZNMqE6uZ0vYOb69tPiY1gKXtzyTjShptwrlv9rdVNqOM9P3ohkk8EuKZqfHHjucylg85yGeVCiCHoLT9x9mpN7ixVHpCagDry37/0RaPNqsu1Mx7OIvdNUsNDk5vUWGRN0yEgPl9AcTxIdVobY+8YTI/lyQSfvzx7cbIVgWQQ3h2Lmlr+6oHBHBrdYNqvfHE8BncywTxKvtz9ZvJpjUWVzbUtq02umFydnKcRDzy2PFwSHA5rdYPc3C8ooPuKceHHGRbqQByCCPRMOeVtr+LExwOHIEobf8x8obJVCLGTDL+xHu5TIpaQ3JzWWn7QMuHIlgB+LraGwQAo7jNo82d3RUNmtn1IVUA8Ra5jKZ6QDZ5w2QkGuLcVOKJ9yYTcea0ETUCsdELJpcP8Iqgadtk1VaY2zu88/BJr8ghl0lRqigfElp7RQ7ZtDbQVXlVAOmAbEajULSyRIYd3TB5dk4jHHryoQjOjHT1UKzWGtxc3mjZq356YozxmJpjBHDdeHJeUTNZNSN9F93xiprmFTWTs33I9QE9q1QA6QBnuWQ31tv3ElJK8kbxQPnKIZvWWFzdYHtnuLX9xRXLKzrIQAcIhdRinQ75NrIoWD5kOCTUaA0riJ6bGmd8n1fk4AThQT2rVADpAJVKaHH/0Rbmdq3l8Bmsh0C9IXlrebgXwNML7aUGsOcYGSaNxnBr+3rBZCoRZyY5cuD7lg85rka2PLkG3X72RmsqgHiGifEY6ZTS9tsZ6A5K27fIG0VGo2HO7cvVbyab0ShX67zzcHOANfMezsZI7VDrh0Fxc4d761stR7XQ5EOqEYi3UEa61asJCStdsBVnTo6RjEeG/suuGybz6WRLrwj2pNFhbqtqrcHNlVJL/8Mhl0kNvQ+5f7XiVgxyjpEKIB2STWvcGnJtXy+YnJsaZzQWbnmOo+0Pc7DdzdVvIzUAXJxNEAmJoZZGb66U2KnLQ9tKjWw7k0XB6uwOyodUAaRDshmNhoQbS8Or7VtSQ+vhs0M2Y6USDqu2f299i9J27dC2GomGuTAz2Lx9r3GYge6wN8doeINt3igyk4wzfcC8omay6cH5kCqAdIjzMBjWnvWjzSr3H20dOnwGK4BsVuvcXSsPoGbeI39IWmozw572rBsmY7EwZydbe0UAJ8djZFIjQ99WndxTg3xWqQDSIadPjpIciQxtD2hvCfcOHopDPiNdN4q2V3Rwrn4z2bTGSqnCaqkygJp5D2eyZaiNV+QwzOuHVWp1Flc2Ovr+nZkY5R8+n+Onn5nse71UAOkQIQTZIdb2nS/u5UP0V4BLs0mi4eHN29cLJuenE4xEW3tFDoPO2/cSjYbkunG4V+SQzaS4tbrBVnX4fMibyxvUGnI38aIdQgj+/Z852zYDsFeoAHIEshmNhUKJ+hBq+7phMqvFmUq0118BYpEQF2aGdwG8TqUGgFx6eKXRe+tblCq1jtsqm7Z9yCGcY9RJCr0bqAByBLJpja2d4dT28x0a6A7Dmre/Xq5iFLc7/qKnxqKcOjE6lG21u198p8F2iDOx8kaR8ViYpybG3K7KY6gAcgSG1Ujf3qmzuLrRsdQA1pd9tVRhpbTdx5p5j71Uy86DrTXHaPi8Nb1gEg4JLs0e7hXBng85rG3VqVc0SFQAOQIXZhJDqe3fXN6g3pBHGj4Pa95+vs0eIK3IZjTuPCizWa31q1qeRDdMLnToFcGeDzlso7VGQ3Y0W98NVAA5ArFIiEuzyaHrAXX7UIThy8TSDZN0aoSJ8VjHn8mmNaSEhSGbY3TQfvGHkcukhs6HfOfhJuVq/chtNQg6CiBCiOeEEDeEEItCiJcOeD8uhPi6/f6bQoizTe99zj5+QwjxgcPKFEKcs8tYtMuM2cefEkJ8WwjxAyHEj4QQHzrOL94tzr7fw7S5jV4wScQjnDnZuf6qjUQ5MzE6dHJfJzPQ95M7NXzS6NpGhSVz+8htlc1YPuSdB8PjQ+6l0Hcuiw6KQwOIECIMfAn4IJAFPi6EyO477ZPAupTyAvAK8LL92SzwApADngO+LIQIH1Lmy8ArdlnrdtkA/zXwmpTyp+wyv9zdr3w8shmNtXKVlSHK28/bqZZH1V+zaW13r4dhYHunzq3V8pGlhkxqhNRodKjkvqPMK2omN4Qj27xRJBISXJh5chM3t+lkBPJeYFFKeVtKWQVeBZ7fd87zwNfs168D7xfWNmzPA69KKStSyjvAol3egWXan3mfXQZ2mR+xX0vAudtSgHG0X7U3OL2AYfmyNxqS6222G21HLpPizlqZcmU4tP0bS5a0ctS2EkLYS7sPjzSqH2G2fjPnpxPEwqGhkpF1w+TCTOde0SDpJICcAt5t+vmefezAc6SUNaAITLb5bKvjk8Aju4z91/pV4G8LIe4BbwB/p4O695zLaStjZFh6QG8/3GSzS/11T9sfjrbaW9fp6FJDNq2xsFSiVm/0ulqeRC+YnDoxyomxzr0isHzIi7PDtX5YN17RoPCTif5x4HeklKeBDwG/J4R4ov5CiE8JIa4JIa6trq72vBLJkShPT44NTQ9o10A/olYNkDs1XJlYeqFIMh7h9MnRI382m9Go1BpDo+3nDbOjVQ0OwtmIaxh8SCsVvtLV928QdBJA7gNnmn4+bR878BwhRARLYlpr89lWx9eAE3YZ+6/1SeA1ACnlvwVGgKn9lZVSfkVKeUVKeWV6erqDX+/oOEb6MKAbJpGQ4OLs0fXXOW2Ek2PRoTGHdcPkckbDUmKPxjDNMdqq1rm92nq/+MPIpofHh/SygQ6dBZDvARft7KgYloF9dd85V4FP2K8/CnxLWt2Dq8ALdpbWOeAi8N1WZdqf+bZdBnaZf2i/fgd4P4AQ4jJWAOn9EKMDsmmNu2ublLZ33Lj8QMkbJhdnk8QjR9dfhRDWAnhDIPfVG5LrhVLXufrPTI8Ti4SGoq0WlkwasvtlOZystWHoxO16RX4dgdh+xGeAbwLXsTKh8kKILwghPmyf9lVgUgixCLwIvGR/No81atCBbwCfllLWW5Vpl/VZ4EW7rEm7bIC/D/ySEOKHwO8D/6F0aQzrSDPDkLffTVpqM7lMioWlEjsB1/bvrpXZ2ql33VbRcIj5ueGYY9TpxkitcFY5Hpa2OnVilNRY1O2qHEjk8FNASvkGlnHdfOzzTa+3gY+1+OwXgS92UqZ9/DZWltb+4zrws53Ut9/sbkVqmPzVsxMu16Z/rJS2WS1VjjUDNpvWqNYa3F4t82wHy5v7leMY6A7ZtMY380tIKbuSwfyCbphoI915RbDnQw7DaC1vFD05A93BTya6Z5jV4kyOxwLfA+o21bKZvbz94LdVNHy8XP1sRmN9c4clM9jrhzlZRccJkrkh2Ihrs1rjzoOyZzOwQAWQrhgWbd/5/brNlgE4NzVOPBIifz/4bXVxJkks0v1Xygm2QW6rekOysGQeabHJg8imNd4OuA95vVBCSu8a6KACSNdk0xpvLW0EWtvPGyZnJkZJjXavv0ZsbT/IwVZKid4DqWF+TkOIYM8xuvNgg+2dxrHbynmoBtmH3PWK1AgkeGQzGtV6g8WVDber0jeOsltcO7KZFPkA5+2vlio82Kge+4s+Ho9wbnI80NLoUfaLb0d2d7QW3LbSDZPUaJRMasTtqrREBZAuCfrmNuVKjTtr5Z4Mn7MZjeLWDkYxmNp+Lwx0h8sBl0Z1wyQWDh17XaeZpOVDBrutrFGtlxMqVADpknNTCUaiocAaeQtLJlL2Jv886MHWeYjNp4+fZZZNa7z7cIviVjC1fb1gcmkuQTR8vEeP40MG9ftXqzdYWCp5dv6HgwogXRIOCebntMBmF/UiA8thfi6JEMHN29cNk6cmxtBGjp+r7wTb6wHsWVteUW9kUbDuzZvLG1RrwfMhbz8oU6k1dueceRUVQI5BNsBr8uQNk5NjUdI90F/HYhHOTY0HdgTSy1z9IO/kuGxWWCtXe5ZVlMukqNYb3FoNng+5NwPduxlYoALIschlNMztGvfWt9yuSs/RC8fP1W8mZxvpQWOjUuPu2mbPetUzyRGmk/FAtpUzWu9VVpHT5sFsK5NYJMQz0+NuV6UtKoAcA+cGDpqR5+ivvcw/z6Y17j/aorgZLG3fkZp6KTUEdd9vZ37LceYVNXNuapzRaDiQo7W8UWR+Lnlsr6jfeLt2Hmd+TiMkgtcDurVaplpr9NTAC+pOcv2QGrIZjcWVUuC0fb1gcnZyjES8oxWUDiUcEsyng7d+WK+9on6iAsgxGI2FeWY6eJvb9FpqgL1eZ9C+7HmjyMR4jFkt3rMycxmNnbrkreVgTZLTu9zZsh3OaC1IPmShuM365o6n18ByUAHkmFh7gwTsoXjfJB4J8cxU7/TX6WScmWQ8eCOQgtnzXP0gSqPm9g5vr232fFmOXCZFKWA+ZC8zIPuNCiDHJJfRMIrbrJerblelZ+gFk/m5JJEe66/OTnJBYafe4K2ljZ5LDWcnxxmLBUvbXyhYo6let9XujPQAtZVeMBHCksi9jgogxyQbsLx9KaUtNfQ+fdDS9jfY3qn3vGw3WFzZoFpv9LynGAoJLgds10tHuuy1LPPsbJJQwNYPyxtFzk2OM94jr6ifqAByTIKWSmgUt3m0udOX4XMuk6LWkIFZP0zfXcKk923laPuNRjC0fd0wmUrEmE72zisCy4c8P50IlIysF6ytkf2ACiDHZDIRZ04bCUwPqJ9baGYDZqTnDZORaIhzU8db1+kgchmNjUqNd9c3e162G+gFk8vp/qzrlA2QNFrc2uHdh1u+MNBBBZCeYK3JE5SHYhEh4HIP1nXaz1MTVgpnUL7seqHI/JxGONSfhyIEY0Z6tdbgreXezitqJkg+pCOF+yGFF1QA6Qm5jMat1XIgtH3dMDk3Nc5YrPf6q6XtJwMh9+3m6vepp3hpNkk4JALRVosrG+zUZd/aaneL6QCoAHuyqLeXMHFQAaQHZNMa9UYw8vbzhtnXmzeb1rgeAG3/3voW5natb1LDSDTMhelEIB6K/TLQHYI0WssbJtPJeM+9on6hAkgPcB64fu8tFjd3uP9oq6/D51wmRbla552H/tb29QFIDUHR9vWCyWg0zNnJ/qzrNDEeI50aCYSMrBf8MQPdQQWQHnD65CjJAGj7g9hCMyh5+3nDJNTnXP1cRmPJ3GZto9K3awwC3TCZTyf74hU5BGH9sEqtzs3lkm8MdFABpCc4eft+7wE59e9nD+jibIJISPh+HxXdMHlmOsFoLNy3awRhRrozr6jfD8Ug+JA3lzeoNfrnFfUDFUB6RDajsbBUou5jbV8vmMz0WX+NR8JcmEn4fgRyfQBSQxBGa/fWtyht1/q+r0U2Y/mQN5b860P6zUAHFUB6RjajsVmt8/Za2e2qdI1u9L+nCP7X9tfLVe4/6n+u/omxGKdOjPq6rfptoDs4D10/j9b0gslYLMzTE2NuV6VjVADpETmf9xa3d+osrmwMZPicy6RYKVVYLflT278+AK/I4bLPtX3d9oqenev9vKJmHB/SzzKybliTLUN99Ip6jQogPeLiTJJoWPj2y764YuuvA9hC0+/afr6Ps/X3k8to3F7dYKvqT21fL5icn04wEu2fVwQghOCyj0e2jcZgvKJeowJIj4hFQlyY8e8kuUFJDdAUQHzaVnrBZE4bYTLR/1z9bEajIWFhyZ9tlR+QLArWvXu94E8f8t31TTYqNV+l8IIKID3Fz8uV64ZJIh7hqQHor6mxKKdPjvpWbujnDPT9+FkafViuUihuD6ytsmmNrZ06d33oQ+Z9aKCDCiA9JZvWeLBRYaW07XZVjkzeMLmcTg5Mf/Vr3v72Tp3F1Y2B9apPnRhFG4n4sq0GnVW0a6T7MNjqhkk4JLg42/uFOfuJCiA9xK+9xUZDDiQttZlcJsWdB2U2q7WBXbMXvLVsSSSDaishhG+z1py5PpcH1FYXZhJEw/5cP0wvmFwYgFfUazoKIEKI54QQN4QQi0KIlw54Py6E+Lr9/ptCiLNN733OPn5DCPGBw8oUQpyzy1i0y4w1vfe3hBC6ECIvhPgX3f7S/cJZw99vX/Z3Hm5SrtYHOoEpm9GQEq4X/JW3v2ugD7CtcpkUC0um77R93TBJp0aYGI8dfnIPiEVCXJxJ+nK0ljeKvjPQoYMAIoQIA18CPghkgY8LIbL7TvsksC6lvAC8ArxsfzYLvADkgOeALwshwoeU+TLwil3Wul02QoiLwOeAn5VS5oC/1/Vv3Se0kShnJvyXt++G/rq7AJ7Pvuy6YZKMRzhzcnC5+tm0xvZOgzsP/LUR1yANdAfLhywipX+C7YONCstmxVcz0B06GYG8F1iUUt6WUlaBV4Hn953zPPA1+/XrwPuFtXPM88CrUsqKlPIOsGiXd2CZ9mfeZ5eBXeZH7Ne/BHxJSrkOIKVcOfqv239y6ZT/HoqFIpEB66+Z1AgnxqK+20nO2RhpkLn6uVP+k0a3d+rcWu39fvGHkc1oPNio+mqOke7CqLZXdBJATgHvNv18zz524DlSyhpQBCbbfLbV8UngkV3G/mtdAi4JIf5UCPEdIcRzB1VWCPEpIcQ1IcS11dXVDn693pLNaNx5UGaj4h9tP2+YXJhJEI8MTn8VQlhGuo8einXHKxrwF/38dIJYOOSrtlpYKtGQkB1wVtHuytg+6sQNYmXnfuEnEz0CXAR+Dvg48FtCiBP7T5JSfkVKeUVKeWV6enrAVdwz0hf8dAMPMC21mZy9flit3hj4tbvh7bUymwP2igCi4RCX5vy1N0g/94tvx7y9k6afgq1umJw6McqJscF4Rb2kkwByHzjT9PNp+9iB5wghIkAKWGvz2VbH14ATdhn7r3UPuCql3LHlsLewAoqn8Ju2v1qqsFKquNL7yWY0KrUGtx/4I29/kDPQ95NLp8gbpm+0fb1QJDkS4fTJ0YFeVxuJ8tTEmK8CSN4o+lK+gs4CyPeAi3Z2VAzLFL+675yrwCfs1x8FviWtO/0q8IKdpXUO64H/3VZl2p/5tl0Gdpl/aL/+37BGHwghprAkrdtH/H37zpw2wsmxKPn7/riBnUDnxgSm3a1IffJl1wsm0bDg0mx/13U6iGxG42G5yrLpD20/b1hp4ZatOVhyGf9srbBZrXH7QdmX8hV0EEBsP+IzwDeB68BrUsq8EOILQogP26d9FZgUQiwCLwIv2Z/NA68BOvAN4NNSynqrMu2yPgu8aJc1aZeNfe6aEELHCjL/QEq5drxfv/cIIchl/GOk6y72qs9PjxOLhHzzZdcNkwszSWKRwSu/e3OMvN9W9YZkoVByrVedTWvcXdv0hQ+5sFRCysFLfb0icvgpIKV8A3hj37HPN73eBj7W4rNfBL7YSZn28dtYWVr7j0us4PRiJ3V2k2xG43f+9C479QbRsLdtprxR5PTJUVJj0YFfOxIOMT/nn7z9vGHyc88O3lcDmG9aP+z9l2ddqUOn3HlQZmun7tqyHE7W2kLB5MrZCVfq0Cl+zsACf5noviGX0ajWG9xa9X7evtt7MDvrh3ld218pbfNgwx2vCCARj3B2cswXwdbtrCJHGvVD2rNeMEmNRjl1YrBeUa9QAaQP+GW12c1qjTsPyq72frJpjfXNHQpFb68f5sYM9P3kMil/PBQNyyu6MOPOuk6zWpyJ8Zjnv3/grlfUC1QA6QPnpsaJR0Ke/7JfLzj6q3srgGZ9svyLF6SGbEbjnYebmNs7rtWhE/JGkUuz7nhF4PiQGvmCt/2iWr3BggvzinqJCiB9IBIOMe+DSXL6AHfWa8X8nIYQ3pcb9ILJmYlRtJHBe0UOzt/puofbSkppzStyOasom9Z4a2mDHQ/PMbrzoEyl1vCtgQ4qgPSNbNpKJfSytq8bRU6MRcmkRlyrw3g8wrnJ8d2VW72KbpjkBrBbYztyPtjJcaVUYa1cdf2hmPWBD+mFDtxxUQGkT+QyGuZ2jfuPttyuSkt0j+iv2Yy39wbZqNS4u+auVwQwnYwzlfC2tr8n9bkcbJ20Zw/Px9INk1gkxPlpf+0B0owKIH3C69p+rd5gYankutQAVlu9+3CL4pY3tf2FgomU7q9VZO0N4m0j3ekIXE4PfrJlM+emEoxEQ57umOQNk2dnk55P9W+Hf2vucebnkp7W9m87+uspDwQQ+8F83aNf9t3Z+h5pq5srJao1b2r7eaPI05NjJF30igDCIcH8nHdnpEspXU+h7wUqgPSJsViEZ6bGPdsD2puB7v4ezLsrqHo02Obvm5wcizKnuecVOeQyGjt1yc0Vb27E5QUD3SHr4TnqJ7o2AAAgAElEQVRGS+Y2D8tVT3RKjoMKIH0km0l5VsLKG0Vbfx13uypMJ+NMJ+OebSvdTrV02ysCb0ujpe0d7q5tum6gOzg+5L117/mQbi4h1EtUAOkjuYzG/UdbPNqsul2VJ9ALJvNzSSIe0V9zHjXSd+oNbiyXXJ0r08zZyXFGo2FPttXCkjUqcjvZwCHr4ay1vGEixN4SNX7FG0+PgOLVG9grufrNZNMaN5dLVGp1t6vyGLdWN6jWGp5pq3BIcDmd9KTc5yVZFKw5RiHhzdGabpicnRwnEe9oOULPogJIH/Gq3FAobrO+ueMZqQGstqo1JDeXvZW379bGSO3IZjSue1DbzxtFJsdjzGpxt6sCwGgszDPTCW8G2wAY6KACSF+ZSsSZ1byn7XthWY79OBKR19oqb5jEIyHOTbnvFTnkMilKlRrvPvSWtu8lr8ghm9Y8l91nbu/wzsNNT33/ukUFkD5jzUj31g28q7/OeecGfnpijPGY97R93fCWVwTN0qh3UlR36g3eWtrw3EPR8SHXy97xIa97sAPXLd75VgSUXCbF4uoG2zve0fb1QpFzk+OMe0h/DYUElz22fthurr5HDHSHZ+eShEPCU221uLJBte4dr8hhd/0wD3VM8h6URbtFBZA+k81o1D2m7esFk8sevHmdJU0aDW9o+/cfWbPjvdZTHImGOT897qmRrRe9IvBmIoteMJlKxJlJuj+v6LioANJnnBvYKzNii1s7vPtwy3NfdLDaaqNS4931TberAnj3oQhWW3npoZg3TEaiIc5NeWtdp8lEnDltxHPB1mudkm5RAaTPPDUxRiIe8cyX3RnKe01qAO/NSN/zitxd1+kgcpkUhaI1m9kL6IUi83Ma4ZB3DHQHZ0a6F6jWGtxcKXmyU9INKoD0mZDH8vb39Fdv6foAF2cTntL29YLJualxxmLe8YocvJQi7swr8upDMZfRPOND3lwpsVOXnuzAdYMKIAMgl0lx3SPavm6Yu0uHeI2RaJiLMwnPjNash6L3Ai14KxPr3voW5nbNs7JMNm35kG8tu79+WJAMdFABZCBk0xqb1TpvP3Rf2/f6BCZnIy63ebRZ5f6jLc+21cnxGJmUN7R93cOyKHhrjpFumIzFwjw96Z15RcdBBZAB4PTM3H4wVmp1bi57W3/NZjSWzQoPNiqu1mN3CXePt5UXHop5wyTksXlFzZw+OUoyHvFMsJ2307CDgAogA+DibIKIB7T9m8sb1BrSs1IDeEfbd65/2aO9arBWe761usFW1V1tXzdMnplOMBoLu1qPVuzOMXJZGm00JNc9LIt2gwogAyAeCXNhxv01eXQPG+gOXsnb1w2TGY96RQ7ZtEZDwg2XtX3dKHp6pAb2+mEFk7qLPuS99S1KFe96Rd2gAsiAyGVS7j8UC7b+OjHmaj3acWIsxqkTo+6PQArezSpyyHlgtLZermIUtz3rfzhkM7YPuVZ2rQ6OhO31++ooqAAyILIZjdVShZXStmt10A2Ty2mNkMf112zGXSN9e6fOzRXvreu0n9MnR0mORFxtq915RR5vq91g62InTi+YhEOCS7Pem1fULSqADIhdacal3mKjIX3RqwarrW4/KLNZrbly/ZvLG9Qb0tNSH4AQwvUZ6Y4s6/URyMWZJNGwcFVG1g2T89PjjES96RV1gwogAyLrcg/o3fVNNio1z3/RweotSrm3w92gcXr0/mirFAuFkmvavl4wmdNGmEx41ysCiEVCXJhJuir35QNmoIMKIAMjNRrl9MlR13pAXp6Bvh+3M7H0gkkiHuEpD3tFDtmMxtZOnTsP3NH28z4w0B1yGfe2VljbqLBket8rOiodBRAhxHNCiBtCiEUhxEsHvB8XQnzdfv9NIcTZpvc+Zx+/IYT4wGFlCiHO2WUs2mXG9l3rbwohpBDiSje/sJvk7J3k3EA3LP314qy3Frs7iFMnRkmNRl0brVleUdLzXhG4m7W2vVPn1mrZ8/6HQzat8WDDHR/SD/OKuuHQACKECANfAj4IZIGPCyGy+077JLAupbwAvAK8bH82C7wA5IDngC8LIcKHlPky8Ipd1rpdtlOXJPB3gTe7+3XdJZtOcWetTLkyeG1fL5hcmE74Qn91tH03eouNhuS6x2frN3NhJkEsHHLFSH9r2ZLO/NJWbmat+WFeUTd0MgJ5L7AopbwtpawCrwLP7zvneeBr9uvXgfcLa1/L54FXpZQVKeUdYNEu78Ay7c+8zy4Du8yPNF3nH2IFGPdSmY5BdlfbH/wN7CepAay2WiiY1OqNgV737YeblKt1X0h9YGn7F2cTrjwU/SSLArt74LjRMdELJpnUCCfHY4ef7CM6CSCngHebfr5nHzvwHCllDSgCk20+2+r4JPDILuOxawkh3gOckVL+7x3U2ZO41QN6sFFh2az4RmoAq60qtcbAtf1dA91nbaUbJlIO1kjXDZNkPMLpk6MDvW63aCNRzkyMuiL35Q3v7WzZC3xhogshQsB/D/z9Ds79lBDimhDi2urqav8rdwTSqRFOjEUH3gNyApafHopuZa3phknEJ16RQzatsVauslIa7PpheaPI5Yz35xU1k0unBt6B26rWub3q/XlF3dBJALkPnGn6+bR97MBzhBARIAWstflsq+NrwAm7jObjSeAvAf+XEOIu8NPA1YOMdCnlV6SUV6SUV6anpzv49QaHEMLqLQ76oejx1VIP4vx0glgkNPAvu14wuTCTIB7xvlfkkHVhtdl6Q7KwVPLVPQVWx+TuWpmNAfqQC0smDRk8Ax06CyDfAy7a2VExLFP86r5zrgKfsF9/FPiWtMbTV4EX7Cytc8BF4LutyrQ/8227DOwy/1BKWZRSTkkpz0opzwLfAT4spbzW5e/tGtm0xsJSiZ0Bavu6YXLqxCgnxvyjv0bDIZ6dHfxGXHkfbjd6OW3NbB6kkf72WpnNat13beXMMboxQB/Sjx24Tjk0gNh+xGeAbwLXgdeklHkhxBeEEB+2T/sqMCmEWAReBF6yP5sHXgN04BvAp6WU9VZl2mV9FnjRLmvSLjswZDMa1VqD26uD0/bzRtF3X3TY2/d7UNr+Smmb1VLFN6awQ3IkytOTYwMd2fp1Y6SsC0Z63jDRRvzjFR2FjvbqlFK+Abyx79jnm15vAx9r8dkvAl/spEz7+G2sLK129fm5TurtRXY3tykUeXYAe21vVmvcflDmF38i0/dr9ZrcKY2vX3uXJXObdKr/X75dr8iHPcVBT5LTCybRsODijL/WdZrTRjg5Fh2o3Kfbo1oryTRY+MJEDxLPTI0Tj4TI3x/MDbywVEL6VH8d9PphfpYasmmNt9c2KW3vDOR6ecPk4kySWMRfjxDLh0wNLNhaXpFJNu2vUW2n+OuvHwAi4RDzc8mByQ1+zMBymE9rCDHAAGKYnD45SmosOpDr9RJnZDuo9cN0H3pFDtmMxo3lwfiQdx5ssL3T8GUHrhNUAHGBrC03DELb1wsmqdEop074T39NxCOcnRwfWG9RN/wzA30/u9r+/f4b6SulbR5sVHzbVrkB+pB5H3fgOkEFEBfIpjWKWzsYxf5PqM/bD0W/6q+DWq68XKlxZ63sOwPdYSYZZ3I8NpC28quB7uAEvkFkremGSSwc4sKMf+YVHQUVQFxgUHn7tXqDhYJ/pQawem7vPNzE7LO2v7BkIqV/e4pCiN2Rbb/ZXdfJp211zvYhByGN6gWTS3MJouFgPmqD+Vt5nPm5JEL0vwd050GZSs3f+qvzQO/3KsZ+9oocshmNm8sbVGv91fZ1w+SpiTG0Ef95RWD7kANYrFNK6WtZtBNUAHGB8XiEc1Pjfe8B6T7ZbrQdg9qKNG+YnBiLkkmN9PU6/SSXSVGtN7i1utHX6+g+Wq24FYOYY7RsVlgrV30ri3aCCiAuMYjlynXDJBYJcX7av/rrTHKEqUS8/21V8LdXBM3afv/aaqNS4+6af/YAaUUuY/mQ9x9t9e0aesF/C3MeFRVAXCKb0bj/aIviZv+0/bxh8uxs0vf6a9ZebbZf1OoNFpZKvpb6wNL2R6PhvrbVQsH07byiZgax66Uz1ytoe4A04+8ni4/Zm5HenxtYShkIqQGsh9XNlVLftP1bq2WqtYbve4rhkGA+neyrtxYEWRT2fMh+SqN6weTs5BiJeEcLfvgSFUBcot+phEvmNg/LVXKn/P1FB6utduqSmyv9mSS3KzUEYLZwv7X9/H2TifEYc5p/vSKAsViEZ6b6O8dI93kGZCeoAOIS08k4M8l433pAfl7XaT/93ogrf9/xisb7Uv4gyWVSlLZr3Fvvj7YfBK/IIZvp394g5vYOb69tBtpABxVAXKWf2r5umAhhLQfid56eHGcsFu5bb1EvmMzPJYn43CuC/q42u1NvcGOpFJhedc72IR9tVnte9kLBGi0HoQPXDv9/Y3xMNq2xuLLB9k6952XnDZOzk+OB0F/DIdG39cMcr8jvprDDs7NJQn3S9m+tblCt+3teUTO7i3X2oa0caToobdUKFUBcJJdJUWtIFld6n7cfFAPdIZdJcd0waTR6q+0bxW0ebe4Epq1GY2HOTyfQ++CtBUkWhf5mYumGyVQixnQy3vOyvYQKIC6yJzf09stubu/wzsPNwEgNYLVVqdJ7bT8IM9D30y9pNG+YjERDPOPjeUXNTCXizGrx/gSQgsnlgHhF7VABxEWenhhjPNb7vP3rAXwo7s1I722wzRtFyyuaC1ZbGcVt1su91fZ1w+TZOY1wKDgPxX4s1lmtNXhruRR4Ax1UAHGVUEhwuQ8z0p0vRC4gUgPApdkk4ZDofVsZJucmxxkPgFfk4KQj9/LBKKW0tkYO0D0FljR6s8c+5OLKBjt1GagOXCtUAHGZbEbjeqG32n7eMJlKxJnxea5+MyPRMOene79+WBBz9fuh7d9/tIW5XQucKZzNaNQbkpvLvfMhh8VABxVAXCeX0ShX67zzcLNnZfp5t7h29Hor0uLmDvfWtwLXVhPjMdKpkZ56a0H0iqA/0qheMBmNhjk76f95RYehAojLOHJDrx6M1VqDmyv+X9fpILJpjSVzm7WNSk/K8/Me6IfRa20/b5iEBFwOkFcEcOaktdRILzsmumEyn04GyitqhQogLnNxNkEkJHrWA7q5UrL01wA+FJ2geL3QmyVN9qSG4JmduYzGrdVyz7R9vWBaizXGwj0pzytYPmSyZ3Jf0OYVHYYKIC4zEg1zYSbRsx5QUKUG2FvVtFfSjF4wmU7GA5mr72j7N5Z6E2wtWTR4gRbsOUY98iHvrW9R2q4FYl21TlABxANk073L288bJmOxYOqvJ8djZFIjPZNmdCO4PcVeZmI92qxy/9FWgNvK8iHf7oEPOUwGOqgA4gmyGY2VUoXV0vG1fWddp6Dqr9keGemVWp3FlY1ASn0AZyZGScYjPRmtBdkrgt5mrem2V/TsXPLYZfkBFUA8wO4NfMzeYqMhuW6YgdT0HbIZjdurG2xVj6ft31zeoNYIbq6+EILLPZqRHmRZFPZ8yF4F2/PTCUaiwfKKWqECiAfIOXLDMb/s99a3KFVqgf2igyUNNCTcWD6eth9kA90hl9FYWCpRP6a2rxsms1qcqUTwvCKAeMTyIXsh9+UDLIsehAogHiA1FuXUidFj94D2NkYK7g3cq424dMNkPBbm6YmxXlTLk2TTGpvVOnfXyscqJ28Ea2HOg+jFHKOH5SqF4nagO3D7UQHEI2Qzx8/bzxsm4ZAItP56+uQo2kjk2KM1Z7G7UEC9IuiNtr+9U2dxdSPQIzWw2mq1VGGltN11GU47B72tmlEBxCPkMhp3HpTZrNa6LkM3TM5PjwdafxVCkM0cb/2wRkMGdrZ+MxdnkkTDx1s/7ObyBvUAe0UOvZhj5CgAlwM+WmtGBRCPkE1rSHm8GzgfcAPdIZtOsbBkdq3tv/Nwk3K1HnhZJhYJcXHmeBtxDUtaai/mGOUNk3RqhInxWK+q5Xk6CiBCiOeEEDeEEItCiJcOeD8uhPi6/f6bQoizTe99zj5+QwjxgcPKFEKcs8tYtMuM2cdfFELoQogfCSH+jRDi6eP84l4jd+p4eftrGxWWzO3APxTBepht7zS486A7bT8/RFJD7piZWHrBJBGPcOZkcL0igNRolNMnR4/XVkNmoEMHAUQIEQa+BHwQyAIfF0Jk9532SWBdSnkBeAV42f5sFngByAHPAV8WQoQPKfNl4BW7rHW7bIAfAFeklD8BvA78Wne/sjfJpEZIjUa73knOGbkEXWqA42/EpReKhEOCi7PB2BipHdmMxoONCitmd9p+3jC5nE4G2ityOE6w3d6pc2s1uPOKWtHJCOS9wKKU8raUsgq8Cjy/75znga/Zr18H3i+srbieB16VUlaklHeARbu8A8u0P/M+uwzsMj8CIKX8tpTSmSr6HeD00X9d7yKEONaMdOdhOgw38IWZBLFwqOvRmm6YXJwZjlz93ay1Ltqq0ZBcLwyHLAqWNHpnrUy5cnQfcmGpREMS2OVeWtFJADkFvNv08z372IHnSClrQBGYbPPZVscngUd2Ga2uBdao5I86qLuvcPL2a/XGkT+rF0wyqRFODoH+Gg2HuDSXOEawDX5aqsPlY2Rivf1wk80h8IocchnLh1zoYv2wvQys4WgrB9+Z6EKIvw1cAf5xi/c/JYS4JoS4trq6OtjKHZNsRqNSa3C7C20/H+DF7g7CGa1JeTQj3UrVrAyF1AegjUR5amKsqwCyO6odkrbaS3s+ujSaN4okRyKcPjna62p5mk4CyH3gTNPPp+1jB54jhIgAKWCtzWdbHV8DTthlPHEtIcTPA78CfFhKeeDCUVLKr0gpr0gpr0xPT3fw63kHRyo46pd9q1rn9urG0HzRwWqrtXKVlSOuH7a7rtNQtVV3c4x0wyQyJF4RQDo1womxaHdtVbBGtZYKPzx0EkC+B1y0s6NiWKb41X3nXAU+Yb/+KPAtaXUNrwIv2Fla54CLwHdblWl/5tt2Gdhl/iGAEOKngP8RK3isdPfreptnpseJRUJHNodvLNv665BIDdC9kb67rtMwtVXammO0cURtP2+YXJhJEI8E3ysCy4fMdTHHqN6QLBRKQ9UpcTg0gNh+xGeAbwLXgdeklHkhxBeEEB+2T/sqMCmEWAReBF6yP5sHXgN04BvAp6WU9VZl2mV9FnjRLmvSLhssySoB/IEQ4s+FEPuDmO+JhkM8O3v0vP1hydVvZt6ebX/U0VreKHLqxCgnxoLvFTk4D7aFI95X+hAZ6A7Z9NF9yDsPymzt1IeurQAih58CUso3gDf2Hft80+tt4GMtPvtF4IudlGkfv42VpbX/+M93Ule/k8tofCO/hJSy4+GwbphoQ6a/JkeinJ0cO3JvUS8Efwb6fpwHW94wuXJ2oqPPrJS2WR0ir8ghl0lRtX3IS7OdLQkU9OXu2+E7Ez3oZDMajzZ3KBQ7z9vP28tyDJv+etT1wzarNe48KA/dF31WizMxHjvSaG1Ys4q6kUbzRpFoWHBhZji8omZUAPEYuSOmXdYbkoUlc2i20Gwml0nx9tompe2djs6/Xigh5fA9FB1t/yjB1jl3mNZ1Anhmapx4JHTkYHtpNkksMnyP0+H7jT3Os3MaQtCxNHPnQZntncbQSQ2wJxl0un7YMGZgOWTTGjeWSux0qO3nDZPTJ0dJjUb7XDNvEQmHmJ9Ldvz9k9JemHPIAq2DCiAeIxGPcHZyfHdlz8MYRgPd4ah5+7pRJDVq7b0ybGQzGtV6g1urGx2df30I13VycKTRTuYYrZQqrJWrQ9tWKoB4kKMsV64XTGLh0FDqrzPJOFOJWOdtZQxnrj7sdTDy9w9vq3Klxp218lDKomAtR/JocwejAx9yb7vf4WwrFUA8SDatcW99i+LW4dq+bphcmksQDQ/fn1IIweV0Z9p+rd5gYWk4c/UBzk0lGIl2tn7YwpI5lF6RgyNHdeKDOArA5XRwN3Frx/A9dXzA3uY27W/gYddfwTLSby5vUK211/ZvPyhTqTWG9qEYDgnm5zpbrHOvVz2cbTU/l0SIzgKIXjB5enKM5MhweUUOKoB4kL1UwvY3sKO/DnMAcbT9xZX22v6wPxTBkUaLh2r7ecPkxFiUdGpkQDXzFuPxCOemxjtK5R32DpwKIB5kJjnCVCJ+aA9o10A/NZz6KzTJDYeM1vJGkVgkxPnp4fOKHLJpDXO7xv1HW23Ps2agD6dX5JDtQBotbe9wd21zaEe1oAKIZ8nZvcV2OAHGWdZjGDk3Nc5oNHx4WxVMnp1NDqVX5JDrYGS76xUNca8aLGn03voWxc3WPqSz7Pswj2qH99vkcbIZjcWVDSq1estz8obJ2SHWX8HW9tPJtqM15RVZzM9phA7R9m+tlqnWGkO5rlMzuynibUYh+ftOCv3wtpUKIB4ll9GoNSQ3l1tr+8O4rtNB5A7J2y8Ut1nf3CF3arjbajQW5pnpRNuHojP/aNjvq06kUb1gMjkeYyYZH1S1PIcKIB7lsFRCc3uHt9c2h75XDdZWpKXtGvfWD9b2h3EJ91Yctm1y/r5JPBLimanxAdbKe0wn48wk422lUacDN8xekQogHuXs5DhjsXDLHtCCvXzHMA+fHQ7LWssbJkLAvAogZDMa9x9t8WizeuD7esFkfi5JZIi9IodspnWw3ak3eGtpuDZxOwh1l3iUUMiaJNeqB6QP2Xaj7ZifS9rafou2KhQ5OzlOIt7R7gWBpt1inVJKJYs2kWvjQy6ubFCtN4Z+VKsCiIfJpjWuF0o0Gk9q+3nDZCox3Pqrw0g0zPk22r6z3aiivbZvFLd5tLkztMty7CebTrX0IZ3R7rArACqAeJhcRmOjUuPd9c0n3tMLJpeHdF2ng8i1kBuKWzu8+3BL9aptJhNx5rSRA9tKeUWP0260phsmI9EQ54bcK1IBxMO00vartQZvLQ/vuk4Hkc1oGMVt1suPa/vXh3gJ91a0WqwzbxQtr2iI5xU189TEGOOxg+cY5Y0i83Ma4dBwd+BUAPEwl2aThEPiiR7Q4soGO3U59MPnZpyVY/dLM3tSgwogDtm0xuLqBts7j2v7umFybmqcceUVAXs+5P57yvGK1D2lAoinGYmGuTCdeKIHNMx7MLei1VakumEylYgzkxzOdZ0OIpfRqDckby0/vhGX8oqexJFGm33Ie+tblLZralSLCiCe56B9v/NGkdFoeOj112YmxmOkU09q+yqr6EmyB2j7xc0d7q1vqVHtPrIZjXK1zjsP93xIZaDvoQKIx8llNJbNCg82KrvHdMNkPp0cev11P/v3/a7U6txcLimpYR9nTo6RjEcea6th3u63HU6Q2N9WIQHPziqvSAUQj7N/Rvpurr6SGp4gm9a4tVre1fZvLm9Qa0jVVvvYm2O091B0pD/VVo9zYSZBJCQek0Z1o8gz0wlGY2EXa+YNVADxOPsXdXP0VzV8fpKsre3fsFdJ1ZWB3pJsRuN6YU/b1wsmM8k402pe0WOMRMNcmEk8JvfpQ7xf/H5UAPE4J8ZinDoxuttbdP5XUsOTOEHVaSO9YDIWC/P0pPKK9pPNaGxW69xdKwP2xkjqnjqQ5rTn9XIVo7itRmo2KoD4gMtpbXeZDt0oElK5+gdy+uQoyZHI7oqyumGt66S8oidpnpFeqdVZXNlQveoWZNMaK6UKq6XKrhKgFAALFUB8QC6jcftBmc1qDb1gcn46wUhU6a/7EULsrjbbaDi5+uqLfhCXZpNEw9Ycoz2vSLXVQTj30PWCuStlXU6rDhyoAOILshkNKa0d0PJKamiLpe2XuLtWZqOicvVbEYuEuDCTJG+Yewa6aqsDcUZrTlvNaSNMJpRXBCqA+ALnBv7Tmw8oFLeV1NCGbFpja6fOG39RAJSB3g5n32/dMBmPhXl6YsztKnmS1FiUUydGrbZSM9AfQwUQH3D65CjaSITX/+wegJIa2uDIDX/w/XuEQ4JLKle/JbmMxmqpwp+8tcrltEZIeUUtyWU0/uztdW6tltVIrYmOAogQ4jkhxA0hxKIQ4qUD3o8LIb5uv/+mEOJs03ufs4/fEEJ84LAyhRDn7DIW7TJjh10j6AghyGY03l6zZsOqG7g1F2YSRMOCt9c2OT89rryiNjj30d21TdWrPgRnI656Q6q2auLQACKECANfAj4IZIGPCyGy+077JLAupbwAvAK8bH82C7wA5IDngC8LIcKHlPky8Ipd1rpddstrDAtOz3o6GWdiPOZybbxLLBLaHXUoA709zR0R1SlpT/O9pBSAPToZgbwXWJRS3pZSVoFXgef3nfM88DX79evA+4W1UcXzwKtSyoqU8g6waJd3YJn2Z95nl4Fd5kcOucZQ4PggY2r266HMatbCiSpXvz3aSHT3tXootqc5wJ4+OepiTbxFJwHkFPBu08/37GMHniOlrAFFYLLNZ1sdnwQe2WXsv1arawwFz9rzPqZU9sehOCO0MxPqi94pF2cTblfB02RSe6s5K69oj8At/C+E+BTwKYCnnnrK5dr0jmxa49N/4zz/3pXg/E794rPPzXNyLMrfmJ9xuyqe5/X/9Gf48f2i8ooOQQjBr330J3ZHtwqLTgLIfeBM08+n7WMHnXNPCBEBUsDaIZ896PgacEIIEbFHGc3nt7rGY0gpvwJ8BeDKlStPbibuU0IhwT/4wLzb1fAF08k4v/IL+206xUFcOTvBlbMTblfDF/ytK2cOP2nI6ETC+h5w0c6OimGZ4lf3nXMV+IT9+qPAt6SU0j7+gp1BdQ64CHy3VZn2Z75tl4Fd5h8ecg2FQqFQuMChIxApZU0I8Rngm0AY+G0pZV4I8QXgmpTyKvBV4PeEEIvAQ6yAgH3ea4AO1IBPSynrAAeVaV/ys8CrQoj/FviBXTatrqFQKBQKdxBB7sRfuXJFXrt2ze1qKBQKha8QQnxfSnnlsPPUTHSFQqFQdIUKIAqFQqHoChVAFAqFQtEVKoAoFJHAcegAAAYGSURBVAqFoitUAFEoFApFVwQ6C0sIsQq83eXHp4AHPaxOr/BivVSdOseL9fJincCb9RqWOj0tpZw+7KRAB5DjIIS41kka26DxYr1UnTrHi/XyYp3Am/VSdXocJWEpFAqFoitUAFEoFApFV6gA0pqvuF2BFnixXqpOnePFenmxTuDNeqk6NaE8EIVCoVB0hRqBKBQKhaIrVAA5ACHEc0KIG0KIRSHESwO43l0hxF8IIf5cCHHNPjYhhPhjIcRN+/+T9nEhhPgNu24/EkK8p6mcT9jn3xRCfKLV9VrU4beFECtCiB83HetZHYQQf8X+HRftz3a0rVuLev2qEOK+3V5/LoT4UNN7n7OvcUMI8YGm4wf+Te0tBd60j3/d3l7gsDqdEUJ8WwihCyHyQoi/63Z7tamT2201IoT4rhDih3a9/pt2ZQlr64ev28ffFEKc7ba+XdTpd4QQd5ra6ift44O838NCiB8IIf612+3UEVJK9a/pH9by8reAZ4AY8EMg2+dr3gWm9h37NeAl+/VLwMv26w8BfwQI4KeBN+3jE8Bt+/+T9uuTR6jDXwfeA/y4H3XA2gfmp+3P/BHwwWPU61eB//KAc7P23ysOnLP/juF2f1PgNeAF+/VvAv9ZB3VKA++xXyeBt+xru9ZeberkdlsJIGG/jgJv2r/XgWUBvwz8pv36BeDr3da3izr9DvDRA84f5P3+IvAvgH/drs0H0U6d/FMjkCd5L7AopbwtpawCrwLPu1CP54Gv2a+/Bnyk6fjvSovvYO3gmAY+APyxlPKhlHId+GPguU4vJqX8v7H2Wel5Hez3NCnld6R1l/9uU1nd1KsVzwOvSikrUso7wCLW3/PAv6ndK3wf8PoBv2O7OhWklH9mvy4B14FTuNheberkdltJKeWG/WPU/ifblNXchq8D77evfaT6dlmnVgzkfhdCnAZ+Afhn9s/t2rzv7dQJKoA8ySng3aaf79H+i9gLJPB/CCG+L6w93QFmpZQF+/USMHtI/fpR717V4ZT9upd1+4wtJ/y2sKWiLuo1CTyS1vbJXdXLlg5+CqsX64n22lcncLmtbFnmz4EVrIfsrTZl7V7ffr9oX7un9/3+Okkpnbb6ot1Wrwgh4vvr1OG1u/37/TrwXwEN++d2bT6QdjoMFUC8wV+TUr4H+CDwaSHEX29+0+7FuJou54U6NPFPgfPATwIF4L9zoxJCiATwvwB/T0ppNr/nVnsdUCfX20pKWZdS/iRwGqsnPD/oOuxnf52EEH8J+BxW3f4qliz12UHVRwjxi8CKlPL7g7pmL1AB5EnuA2eafj5tH+sbUsr79v8rwP+K9SVbtofC2P+vHFK/ftS7V3W4b7/uSd2klMv2A6AB/BZWe3VTrzUsOSKy7/ihCCGiWA/qfy6l/Jf2YVfb66A6eaGtHKSUj4BvAz/Tpqzd69vvp+xr9+W+b6rTc7YMKKWUFeB/ovu26ubv97PAh4UQd7HkpfcB/wMeaaeWdGOcBPkf1j7xt7EMKMdsyvXxeuNAsun1/4flXfxjHjdkf81+/Qs8buh91z4+AdzBMvNO2q8njliXszxuVvesDjxpKn7oGPVKN73+L7A0X4AcjxuIt7HMw5Z/U+APeNyk/OUO6iOwdO1f33fctfZqUye322oaOGG/HgX+H+AXW5UFfJrHzeHXuq1vF3VKN7XlrwP/yKX7/efYM9Fda6eO6nrcAoL4Dyvr4i0srfZX+nytZ+w/5g+BvHM9LD3z3wA3gf+z6cYUwJfsuv0FcKWprP8YyzRbBP6jI9bj97Ekjh0sffSTvawDcAX4sf2Zf4I9ibXLev2efd0fAVd5/CH5K/Y1btCU+dLqb2q3/3ft+v4BEO+gTn8NS576EfDn9r8Pudleberkdlv9BPAD+/o/Bj7frixgxP550X7/mW7r20WdvmW31Y+B/5m9TK2B3e/2Z3+OvQDiWjt18k/NRFcoFApFVygPRKFQKBRdoQKIQqFQKLpCBRCFQqFQdIUKIAqFQqHoChVAFAqFQtEVKoAoFAqFoitUAFEoFApFV6gAolAoFIqu+P8BnvYauUGfA34AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(learn_rates)\n",
    "print(train_accs[-4])\n",
    "accs_file = open('train_accs.txt', 'w')\n",
    "for item in train_accs:\n",
    "    accs_file.write(\"%s\\n\" % item)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Implements CLR\n",
    "class ChannelSystem(object):\n",
    "    \"\"\"\n",
    "    Generates a channel coding model with encoder + channel + decoder.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, config, train_size, test_size):\n",
    "        self.config = config\n",
    "        self.training_counter = 0\n",
    "        self.test_counter = 0\n",
    "        #self.train_data = train_data\n",
    "        #self.test_data = test_data\n",
    "        self.train_size = train_size\n",
    "        self.test_size = test_size \n",
    "        # ==== reset graph ====\n",
    "        tf.reset_default_graph()\n",
    "          \n",
    "        self.chan_enc_in_len = config.chan_enc_in_len\n",
    "        self.chan_dec_out_len = config.chan_dec_out_len\n",
    "        \n",
    "        #===Legacy placeholders====\n",
    "        #self.chan_enc_inputs = tf.placeholder_with_default(queue_vars['enc_inputs'],shape=(config.batch_size, None), name='encoder_inputs')\n",
    "        #self.chan_enc_inputs_len = tf.placeholder_with_default(queue_vars['enc_inputs_len'],shape=(None,), name='encoder_inputs_length')\n",
    "        #self.dec_inputs = tf.placeholder_with_default(queue_vars['dec_inputs'],shape=(config.batch_size, None), name='decoder_targets')\n",
    "        #self.dec_targets = tf.placeholder_with_default(queue_vars['dec_targets'],shape=(config.batch_size, None), name='decoder_targets')\n",
    " \n",
    "        \n",
    "        # ======= Placeholders =======        \n",
    "        self.chan_enc_inputs = tf.placeholder(tf.float32, shape = [None, self.chan_enc_in_len], name = 'enc_inputs')\n",
    "        self.dec_targets = tf.placeholder(tf.float32, shape = [None, self.chan_dec_out_len], name = 'dec_targets')\n",
    "        self.isTrain = tf.placeholder(tf.bool,shape=(), name='isTrain')\n",
    "\n",
    "        self.learn_rate = tf.placeholder(tf.float32,shape=(), name='learn_rate')\n",
    "        \n",
    "        self.X = tf.placeholder(tf.float32, shape = [None, self.chan_enc_in_len], name = 'dataset_iter_X')\n",
    "        self.Y = tf.placeholder(tf.float32, shape = [None, self.chan_dec_out_len], name = 'dataset_iter_Y')\n",
    "        print(\"X:\", self.X)  \n",
    "        self.dataset = tf.data.Dataset.from_tensor_slices((self.X, self.Y)).shuffle(buffer_size=100).batch(config.batch_size).repeat()\n",
    "        \n",
    "        self.iterator = self.dataset.make_initializable_iterator()\n",
    "        \n",
    "        self.chan_enc_inputs, self.dec_targets = self.iterator.get_next()\n",
    "        # ==== Building neural network graph ====\n",
    "        \n",
    "        \n",
    "        self.chan_encoder = ChannelEncoder(self.chan_enc_inputs, \n",
    "                                       self.isTrain,\n",
    "                                       self.config)\n",
    "\n",
    "        self.channel = Channel(self.chan_encoder.output,\n",
    "                               self.config)\n",
    "        \n",
    "        self.decoder = ChannelDecoder(self.channel.output,\n",
    "                                     self.isTrain,\n",
    "                                     self.config)\n",
    "\n",
    "        \n",
    "        # ==== define loss and training op and accuracy ====\n",
    "        \n",
    "        self.accuracy = self.define_accuracy()\n",
    "        self.loss, self.train_op = self.define_loss()\n",
    "\n",
    "        # ==== set up training/updating procedure ====\n",
    "        self.saver = tf.train.Saver(max_to_keep = self.config.save_max)\n",
    "\n",
    "\n",
    "#     def load_trained_model(self, sess, model_save_path):\n",
    "#         \"\"\"\n",
    "#         Loads a trained model from what was saved. Insert the trained model path\n",
    "#         \"\"\"\n",
    "#         meta_file_path = model_save_path + \".meta\"\n",
    "        \n",
    "#         new_saver = tf.train.import_meta_graph(meta_file_path) #Loads graph\n",
    "#         new_saver.restore(sess, tf.train.latest_checkpoint('checkpoints/'))\n",
    "        \n",
    "#         #Test\n",
    "#         variables_names = [v.name for v in tf.trainable_variables()]\n",
    "#         values = sess.run(variables_names)\n",
    "#         print(\"Printing restored variables:\")\n",
    "#         for k, v in zip(variables_names, values):\n",
    "#             print (\"Variable: \", k)\n",
    "#             print (\"Shape: \", v.shape)\n",
    "#             print (v)\n",
    "\n",
    "    def define_accuracy(self):\n",
    "        eq_indicator = tf.cast(tf.equal(self.decoder.bin_pred, self.dec_targets), dtype=tf.float32)\n",
    "        return tf.reduce_mean(eq_indicator)\n",
    "\n",
    "    def define_loss(self):\n",
    "        \n",
    "        adapted_targets = tf.cast(self.dec_targets > 0, tf.float32)\n",
    "        loss = tf.losses.hinge_loss(adapted_targets, self.decoder.dec_network_out)\n",
    "\n",
    "        # train it\n",
    "        train_op = tf.train.AdamOptimizer(learning_rate=self.learn_rate).minimize(loss)\n",
    "        return loss, train_op\n",
    "    \n",
    "    \"\"\"\n",
    "    Script to generate train and test data\n",
    "    \"\"\"\n",
    "    def gen_bin_data(self, num_samples, sample_len, seed_num = None):\n",
    "        \n",
    "        if (seed_num != None):\n",
    "            np.random.seed(seed_num)\n",
    "        randMat = np.random.rand(num_samples, sample_len)\n",
    "        binMat = np.where(randMat > 0.5, 1.0, -1.0)\n",
    "        return binMat\n",
    "\n",
    "          \n",
    "    def train_and_test(self, sess, grammar=None):\n",
    "        \"\"\"\n",
    "        Trains the model\n",
    "        \"\"\"\n",
    "        try:\n",
    "            n_epochs = self.config.epochs\n",
    "            batch_sz = self.config.batch_size\n",
    "            max_batch = self.config.max_batch_in_epoch\n",
    "\n",
    "            sess.run(tf.global_variables_initializer())\n",
    "\n",
    "            #Train on train data\n",
    "            print(\"Training...\")\n",
    "            tic = time.time()\n",
    "\n",
    "            accuracies = []\n",
    "            learn_rates = []\n",
    "        \n",
    "            #train_data_gen = self.gen_bin_data(self.train_size, self.chan_enc_in_len)\n",
    "\n",
    "            max_lr = self.config.learn_rate\n",
    "            base_lr = (1e-7)\n",
    "            step_size = 5 * self.config.max_batch_in_epoch\n",
    "            \n",
    "            for epoch in range(n_epochs):\n",
    "                train_data = self.gen_bin_data(self.train_size, self.chan_enc_in_len)\n",
    "                sess.run(self.iterator.initializer, feed_dict = {self.X: train_data, self.Y: train_data})\n",
    "                len_acc = len(accuracies)\n",
    "                \n",
    "\n",
    "                for i in range(max_batch):               \n",
    "                    cycle = np.floor(1+self.training_counter/(2*step_size))\n",
    "                    x = np.abs(self.training_counter/step_size - 2*cycle + 1)\n",
    "                    curr_lr = max_lr - (max_lr-base_lr)*np.maximum(0, (1-x))\n",
    "                    \n",
    "                    learn_rates.append(curr_lr)\n",
    "                    train_fd = {self.isTrain: True, self.learn_rate: curr_lr}       \n",
    "                    _, loss = sess.run([self.train_op, self.loss], feed_dict = train_fd)\n",
    "\n",
    "                    self.training_counter += 1\n",
    "\n",
    "                    if (self.training_counter % self.config.iter_print_rate == 0):\n",
    "\n",
    "                        toc = time.time()\n",
    "\n",
    "                        acc = sess.run(self.accuracy, train_fd)\n",
    "\n",
    "                        print(\"Epoch: \", epoch + 1, \n",
    "                              \"Accuracy: \", acc,\n",
    "                              \"Training iteration: \", self.training_counter,\n",
    "                              \"Training time: \", int(toc-tic), \"s\",\n",
    "                              \"Training loss: \", loss,\n",
    "                              \"Learning rate\", curr_lr)\n",
    "                        accuracies.append(acc)\n",
    "\n",
    "                    if self.training_counter % self.config.iter_save_rate == 0:\n",
    "                        if (len(accuracies) > 1 and accuracies[-1] > np.amax(accuracies[:-1])):\n",
    "                            self.saver.save(sess, self.config.model_save_path, global_step=self.training_counter)\n",
    "                            print(\"Model saved in file: %s\" % self.config.model_save_path)\n",
    "\n",
    "        except KeyboardInterrupt:\n",
    "            print(\"Training interrupted\")\n",
    "            \n",
    "        #Save model, plot accuracies    \n",
    "        self.saver.save(sess, self.config.model_save_path, global_step=self.training_counter)\n",
    "        print(\"Model saved in file: %s\" % self.config.model_save_path) \n",
    "        \n",
    "        plt.plot(accuracies)\n",
    "        plt.title(\"Accuracies vs epochs\")\n",
    "        plt.xlabel('epochs')\n",
    "        plt.ylabel('Acc')\n",
    "        plt.grid(True)               \n",
    "            \n",
    "        #Test on test data    \n",
    "        test_data = self.gen_bin_data(self.test_size, self.chan_enc_in_len)\n",
    "\n",
    "        sess.run(self.iterator.initializer, feed_dict = {self.X: test_data, self.Y: test_data})\n",
    "        print(\"Testing...\")\n",
    "        \n",
    "        test_fd = {self.isTrain: False, self.learn_rate: self.config.learn_rate} \n",
    "        test_loss = sess.run(self.loss, feed_dict = test_fd)\n",
    "        \n",
    "        test_acc = sess.run(self.accuracy, test_fd)\n",
    "        \n",
    "        print(\"Test loss: \", test_loss, \"\\nTest accuracy: \", test_acc)\n",
    "            \n",
    "        return (accuracies, test_acc, learn_rates)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'code_rates' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-10-a4d04c405972>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msubplot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcode_rates\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_accs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtitle\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Accuracy after \"\u001b[0m\u001b[0;34m+\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnum_epochs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m+\u001b[0m \u001b[0;34m\" epochs vs code rate\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mylabel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Training accuracy\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgrid\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'code_rates' is not defined"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAACGCAYAAADQHI0rAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvFvnyVgAAC6lJREFUeJzt3X+IHPd9xvH3EzmyqZMmSnUFo1+WqVJbTUrsLKpLoElJLCv+QwqkbWQwkYObAzdKISkFl0BdZAL5QRsIqLWvrUhSqOXEf5QrdRAmtjGEKNEKO46louSsptZdA1Yix/8okSv56R8z7q0uJ+3c3t7O6b7PC5ab+c58h8992dvn5sfOyDYREVGuN7RdQEREtCtBEBFRuARBREThEgQREYVLEEREFC5BEBFRuL5BIOmApJckPX+J5ZL0ZUlTkp6TdEvPsj2SflS/9gyz8IiIGI4mewRfAXZcZvkHgS31axz4BwBJbwPuB34P2AbcL2nNYoqNiIjh6xsEtp8GzlxmlV3A11w5DLxV0nXA7cDjts/Yfhl4nMsHSkREtGAY5wjWAad65qfrtku1R0TEMnJV2wUASBqnOqzEtdde++4bb7yx5YoiIq4sR48e/antsUH6DiMIZoANPfPr67YZ4H1z2p+abwO2J4AJgE6n4263O4SyIiLKIem/B+07jENDk8BH66uHbgVesf0T4BCwXdKa+iTx9rotIiKWkb57BJIepvrPfq2kaaorgd4IYPtB4DHgDmAKOAt8rF52RtIDwJF6U/tsX+6kc0REtKBvENi+s89yA5+4xLIDwIHBSouIiFHIN4sjIgqXIIiIKFyCICKicAmCiIjCJQgiIgqXIIiIKFyCICKicAmCiIjCJQgiIgqXIIiIKFyCICKicAmCiIjCJQgiIgqXIIiIKFyCICKicAmCiIjCNQoCSTsknZA0Jem+eZZ/SdKz9euHkn7es+xCz7LJYRYfERGL1+RRlauA/cBtwDRwRNKk7eOvr2P7Uz3rfxK4uWcTv7D9ruGVHBERw9Rkj2AbMGX7pO1XgYPArsusfyfw8DCKi4iIpdckCNYBp3rmp+u2XyFpE7AZeKKn+RpJXUmHJX1o4EojImJJ9D00tEC7gUdtX+hp22R7RtINwBOSfmD7hd5OksaBcYCNGzcOuaSIiLicJnsEM8CGnvn1ddt8djPnsJDtmfrnSeApLj5/8Po6E7Y7tjtjY2MNSoqIiGFpEgRHgC2SNktaTfVh/ytX/0i6EVgDfKenbY2kq+vptcB7gONz+0ZERHv6HhqyfV7SXuAQsAo4YPuYpH1A1/brobAbOGjbPd1vAh6S9BpV6Hyu92qjiIhony7+3G5fp9Nxt9ttu4yIiCuKpKO2O4P0zTeLIyIKlyCIiChcgiAionAJgoiIwiUIIiIKlyCIiChcgiAionAJgoiIwiUIIiIKlyCIiChcgiAionAJgoiIwiUIIiIKlyCIiChcgiAionAJgoiIwjUKAkk7JJ2QNCXpvnmW3y3ptKRn69ef9izbI+lH9WvPMIuPiIjF6/uoSkmrgP3AbcA0cETS5DyPnHzE9t45fd8G3A90AANH674vD6X6iIhYtCZ7BNuAKdsnbb8KHAR2Ndz+7cDjts/UH/6PAzsGKzUiIpZCkyBYB5zqmZ+u2+b6sKTnJD0qacNC+koal9SV1D19+nTD0iMiYhiGdbL434Hrbf8u1X/9X11IZ9sTtju2O2NjY0MqKSIimmgSBDPAhp759XXb/7P9M9vn6tl/At7dtG9ERLSrSRAcAbZI2ixpNbAbmOxdQdJ1PbM7gf+spw8B2yWtkbQG2F63RUTEMtH3qiHb5yXtpfoAXwUcsH1M0j6ga3sS+HNJO4HzwBng7rrvGUkPUIUJwD7bZ5bg94iIiAHJdts1XKTT6bjb7bZdRkTEFUXSUdudQfrmm8UREYVLEEREFC5BEBFRuARBREThEgQREYVLEEREFC5BEBFRuARBREThEgQREYVLEEREFC5BEBFRuARBREThEgQREYVLEEREFC5BEBFRuEZBIGmHpBOSpiTdN8/yT0s6Xj+8/luSNvUsuyDp2fo1ObdvRES0q+8TyiStAvYDtwHTwBFJk7aP96z2DNCxfVbSvcAXgI/Uy35h+11DrjsiIoakyR7BNmDK9knbrwIHgV29K9h+0vbZevYw1UPqIyLiCtAkCNYBp3rmp+u2S7kH+GbP/DWSupIOS/rQADVGRMQS6ntoaCEk3QV0gPf2NG+yPSPpBuAJST+w/cKcfuPAOMDGjRuHWVJERPTRZI9gBtjQM7++bruIpA8AnwF22j73ervtmfrnSeAp4Oa5fW1P2O7Y7oyNjS3oF4iIiMVpEgRHgC2SNktaDewGLrr6R9LNwENUIfBST/saSVfX02uB9wC9J5kjIqJlfQ8N2T4vaS9wCFgFHLB9TNI+oGt7Evgi8CbgG5IAXrS9E7gJeEjSa1Sh87k5VxtFRETLZLvtGi7S6XTc7XbbLiMi4ooi6ajtziB9883iiIjCJQgiIgqXIIiIKFyCICKicAmCiIjCJQgiIgqXIIiIKFyCICKicAmCiIjCJQgiIgqXIIiIKFyCICKicAmCiIjCJQgiIgqXIIiIKFyCICKicI2CQNIOSSckTUm6b57lV0t6pF7+XUnX9yz7q7r9hKTbh1d6REQMQ98gkLQK2A98ENgK3Clp65zV7gFetv1bwJeAz9d9t1I94/h3gB3A39fbi4iIZaLJHsE2YMr2SduvAgeBXXPW2QV8tZ5+FHi/qocX7wIO2j5n+7+AqXp7ERGxTDQJgnXAqZ756bpt3nVsnwdeAX6jYd+IiGjRVW0XACBpHBivZ89Jer7NepaRtcBP2y5imchYzMpYzMpYzPrtQTs2CYIZYEPP/Pq6bb51piVdBbwF+FnDvtieACYAJHVtd5r+AitZxmJWxmJWxmJWxmKWpO6gfZscGjoCbJG0WdJqqpO/k3PWmQT21NN/BDxh23X77vqqos3AFuB7gxYbERHD13ePwPZ5SXuBQ8Aq4IDtY5L2AV3bk8A/A/8iaQo4QxUW1Ot9HTgOnAc+YfvCEv0uERExgEbnCGw/Bjw2p+2ve6Z/CfzxJfp+FvjsAmqaWMC6K13GYlbGYlbGYlbGYtbAY6HqCE5ERJQqt5iIiChca0GwmNtWrDQNxuLTko5Lek7StyRtaqPOUeg3Fj3rfViSJa3YK0aajIWkP6nfG8ck/euoaxyVBn8jGyU9KemZ+u/kjjbqXGqSDkh66VKX2Kvy5XqcnpN0S6MN2x75i+qk8wvADcBq4PvA1jnr/BnwYD29G3ikjVqXyVj8IfBr9fS9JY9Fvd6bgaeBw0Cn7bpbfF9sAZ4B1tTzv9l23S2OxQRwbz29Ffhx23Uv0Vj8AXAL8Pwllt8BfBMQcCvw3SbbbWuPYDG3rVhp+o6F7Sdtn61nD1N9H2MlavK+AHiA6n5WvxxlcSPWZCw+Duy3/TKA7ZdGXOOoNBkLA79eT78F+J8R1jcytp+mujLzUnYBX3PlMPBWSdf1225bQbCY21asNAu9Dcc9VIm/EvUdi3pXd4Pt/xhlYS1o8r54O/B2Sd+WdFjSjpFVN1pNxuJvgLskTVNd4fjJ0ZS27Ax0W59lcYuJaEbSXUAHeG/btbRB0huAvwPubrmU5eIqqsND76PaS3xa0jtt/7zVqtpxJ/AV238r6fepvtf0DtuvtV3YlaCtPYKF3LaCObetWGka3YZD0geAzwA7bZ8bUW2j1m8s3gy8A3hK0o+pjoFOrtATxk3eF9PApO3/dXV33x9SBcNK02Qs7gG+DmD7O8A1VPchKk2jz5O52gqCxdy2YqXpOxaSbgYeogqBlXocGPqMhe1XbK+1fb3t66nOl+y0PfA9VpaxJn8j/0a1N4CktVSHik6OssgRaTIWLwLvB5B0E1UQnB5plcvDJPDR+uqhW4FXbP+kX6dWDg15EbetWGkajsUXgTcB36jPl79oe2drRS+RhmNRhIZjcQjYLuk4cAH4S9srbq+54Vj8BfCPkj5FdeL47pX4j6Okh6nCf219PuR+4I0Ath+kOj9yB9WzX84CH2u03RU4VhERsQD5ZnFEROESBBERhUsQREQULkEQEVG4BEFEROESBBERhUsQREQULkEQEVG4/wMG8siEiS1dnQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.subplot(2,1,1)\n",
    "plt.plot(code_rates, train_accs)\n",
    "plt.title(\"Accuracy after \"+ str(num_epochs)+ \" epochs vs code rate\")\n",
    "plt.ylabel(\"Training accuracy\")\n",
    "plt.grid(True)\n",
    "\n",
    "plt.subplot(2,1,2)\n",
    "plt.plot(code_rates, test_accs)\n",
    "plt.ylabel(\"Testing accuracy\")\n",
    "plt.grid(True)\n",
    "plt.xlabel(\"Code rate\")\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
